Merge pull request #34 from tamnd/release-0.3.3

Cut the v0.3.3 release notes

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
         id: chrome
       - name: gofmt
         run: |
-          unformatted=$(gofmt -s -l asset browser cli clone cmd robots sanitize urlx)
+          unformatted=$(gofmt -s -l .)
           if [ -n "$unformatted" ]; then
             echo "These files need gofmt -s -w:"
             echo "$unformatted"
@@ -48,9 +48,15 @@ jobs:
         run: go vet ./...
       - name: build
         run: go build ./...
+      # The GitHub Ubuntu runner disables unprivileged user namespaces (AppArmor),
+      # so Chrome's sandbox cannot initialize there and the secure default would
+      # make it refuse to start. IN_DOCKER is kage's documented escape hatch for
+      # exactly that case, and setting it here also exercises the container path.
+      # macOS does not need it, so it stays empty on that leg.
       - name: test
         env:
           KAGE_CHROME: ${{ steps.chrome.outputs.chrome-path }}
+          IN_DOCKER: ${{ matrix.os == 'ubuntu-latest' && '1' || '' }}
         run: go test -race -count=1 -coverprofile=coverage.out ./...
       - name: coverage summary
         if: matrix.os == 'ubuntu-latest'
@@ -101,3 +107,40 @@ jobs:
         run: |
           go mod tidy
           git diff --exit-code -- go.mod go.sum
+
+  # Compile the optional native-window viewer (-tags webview, cgo) so that path
+  # keeps building. The default CI build is pure Go and never touches it. The
+  # viewer code is the same Go on every OS, only the system WebView library
+  # differs, so a macOS compile (WebKit ships in the SDK) catches our
+  # regressions without the WebKitGTK version juggling Linux runners need. It is
+  # build-only: actually opening a window needs a display.
+  webview:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+          check-latest: true
+          cache: true
+      - name: build webview viewer
+        run: CGO_ENABLED=1 go build -tags webview ./cmd/kage
+
+  # Cross-compile the GUI-subsystem Windows base the release ships for
+  # double-click viewers (kage pack --base). A change that breaks the
+  # -H windowsgui link is caught here instead of at release time. Pure Go, so it
+  # cross-compiles from Linux with no extra toolchain.
+  windows-gui:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+          check-latest: true
+          cache: true
+      - name: build windowsgui base
+        env:
+          GOOS: windows
+          CGO_ENABLED: "0"
+        run: go build -ldflags "-H=windowsgui" -o kage-windowsgui.exe ./cmd/kage

.goreleaser.yaml

@@ -43,9 +43,35 @@ builds:
       - freebsd_amd64
       - freebsd_arm64
 
+  # A second Windows build linked for the GUI subsystem (-H windowsgui). It is
+  # the same pure-Go kage, but a viewer packed onto it shows only its window when
+  # double-clicked, with no console flashing behind it. Users point `kage pack
+  # --base` at this to build a clean double-click Windows app. The default build
+  # above stays console-attached so the CLI still prints clone progress.
+  - id: kage-gui
+    binary: kage
+    main: ./cmd/kage
+    env:
+      - CGO_ENABLED=0
+    flags:
+      - -trimpath
+    ldflags:
+      - -s -w
+      - -H=windowsgui
+      - -X github.com/tamnd/kage/cli.Version={{ .Version }}
+      - -X github.com/tamnd/kage/cli.Commit={{ .ShortCommit }}
+      - -X github.com/tamnd/kage/cli.Date={{ .CommitDate }}
+    mod_timestamp: "{{ .CommitTimestamp }}"
+    targets:
+      - windows_amd64
+      - windows_arm64
+
 archives:
-  # tar.gz everywhere except a zip on Windows.
+  # tar.gz everywhere except a zip on Windows. Scoped to the console build so
+  # this is the archive every package manager installs.
   - id: default
+    ids:
+      - kage
     name_template: "kage_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}"
     format_overrides:
       - goos: windows
@@ -54,6 +80,18 @@ archives:
       - LICENSE
       - README.md
 
+  # A separate zip for the GUI-subsystem Windows binary, named so it cannot be
+  # confused with the console build. It is a base to pack against, not something
+  # to run directly, so no package manager points at it.
+  - id: windows-gui
+    ids:
+      - kage-gui
+    name_template: "kage_{{ .Version }}_windows-gui_{{ .Arch }}"
+    formats: [zip]
+    files:
+      - LICENSE
+      - README.md
+
 nfpms:
   # One nfpm definition emits the deb, rpm, and apk for every Linux build. kage
   # is a user command, not a daemon, so there is no unit file and no
@@ -61,6 +99,8 @@ nfpms:
   # dependency the user supplies (or the container image bundles).
   - id: linux-packages
     package_name: kage
+    ids:
+      - kage
     file_name_template: "{{ .ConventionalFileName }}"
     vendor: tamnd
     homepage: https://github.com/tamnd/kage
@@ -106,6 +146,8 @@ homebrew_casks:
   # HOMEBREW_TAP_GITHUB_TOKEN (a PAT with write to tamnd/homebrew-tap) is set,
   # so a tokenless release still writes the cask into dist for inspection.
   - name: kage
+    ids:
+      - default
     repository:
       owner: tamnd
       name: homebrew-tap
@@ -119,8 +161,11 @@ homebrew_casks:
       email: tamnd87@gmail.com
 
 scoops:
-  # Scoop manifest for Windows, pushed to the bucket repository.
-  - repository:
+  # Scoop manifest for Windows, pushed to the bucket repository. It installs the
+  # console build (the CLI), not the GUI base.
+  - ids:
+      - default
+    repository:
       owner: tamnd
       name: scoop-bucket
       token: '{{ envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}'

CHANGELOG.md

@@ -6,6 +6,180 @@ All notable changes to kage are recorded here. The format follows
 
 ## [Unreleased]
 
+## [0.3.3] - 2026-06-16
+
+### Fixed
+
+- Chrome no longer downloads a file to your Downloads folder when a crawl follows a link that turns out to be a binary (reported in #32).
+  An extensionless link is queued as a page, so the page worker navigated to it in Chrome, and a link that served a zip or a CSV made Chrome save the file to `~/Downloads`, a surprise side effect of a clone.
+  kage now denies Chrome-initiated downloads browser-wide, since every asset is fetched through kage's own downloader, and detects a navigation whose response is not HTML and reroutes that URL to the asset downloader, where the size and media policy decides whether to localise it or leave it on the live web.
+
+## [0.3.2] - 2026-06-16
+
+### Fixed
+
+- Saved pages now declare their character encoding, so text no longer mojibakes in a reader.
+  kage writes every page as UTF-8, but a source that set its charset only in the HTTP `Content-Type` header, with no `<meta charset>` in the markup, lost that signal once the page became a standalone file.
+  A reader serving the bytes without a charset then fell back to its locale encoding and turned every curly quote, dash, and non-breaking space into mojibake (reported in #16 and #29).
+  kage now inserts a `<meta charset="utf-8">` at the top of `<head>` when the page does not already declare one, so the page is self-describing in any reader.
+
+## [0.3.1] - 2026-06-15
+
+### Fixed
+
+- A served mirror whose entry point is a nested page no longer loses its CSS and
+  images when opened at the root. kage saves each page's asset links as
+  mirror-relative paths (`../_kage/...`) computed for that page's own location,
+  but the viewer answered `/` with the main page's bytes in place, so the browser
+  resolved those relative URLs against `/` and missed every one. A
+  `developer.apple.com/documentation` mirror, whose main page is
+  `developer.apple.com/documentation/index.html`, landed at `/` completely
+  unstyled. kage now redirects `/` to the main page's canonical content path, the
+  way the archive's `W/mainPage` redirect already does, so the browser resolves
+  the page's relative assets correctly. Kiwix was unaffected because it follows
+  that redirect itself.
+
+## [0.3.0] - 2026-06-15
+
+### Added
+
+- `kage parquet export <file.zim>` and `kage parquet import <file.parquet>`
+  convert a packed archive to a columnar Parquet table and back. The table is
+  flat, one row per archive entry, with clear columns (url, host, title, mime,
+  extracted text, content), so it drops straight into the tooling a dataset host
+  like Hugging Face expects, and DuckDB or pandas can query it as is. The column
+  names follow the open-index/open-markdown dataset (`doc_id`, `url`, `host`,
+  `crawl_date`, `content_length`, `text_length`, `text`), with `doc_id` a
+  deterministic UUID v5 of the page URL, so a kage export sits alongside other
+  web-crawl datasets. The conversion is lossless: a ZIM round-tripped through
+  Parquet reproduces every entry, its metadata, and the main page byte for byte.
+- `kage pack --incremental` keeps a small cache sidecar next to the output and
+  reuses the compression of any cluster whose bytes have not changed since the
+  last pack. Compressing clusters with zstd is the dominant cost of packing a
+  large mirror, so re-packing after a small change (a `--refresh`, a handful of
+  edited pages) only compresses what actually changed instead of the whole
+  archive. A cached cluster is byte-for-byte what a fresh compression produces,
+  so the archive stays deterministic and valid. The pack reports how many
+  clusters it reused versus compressed.
+- Identical pages are now stored once. When a rendered page's bytes match a page
+  already written, kage stores it as a hard link to the first copy instead of a
+  second full file. This collapses the duplicate content a faceted site spawns
+  when many `?q=…`/`?page=…` URLs all render the same page. The final summary
+  reports how many pages were deduped this way.
+
+### Changed
+
+- Clones no longer pull a site's bulk downloads into the mirror by default. Video
+  and audio, installers and disk images (`.dmg`, `.pkg`, `.exe`, `.msi`, ...),
+  archives, and PDFs are left pointing at their live URL instead of downloaded,
+  because they are rarely needed to read a site offline yet routinely make up
+  most of its bytes (a developer.apple.com crawl was 18 of 19 GB of such assets).
+  Page-rendering assets (images, fonts, CSS) are unaffected. `--keep-media`
+  restores the old behavior, and `--skip-ext .foo` leaves more extensions remote.
+- Assets are localized only from the seed's own registrable domain by default.
+  developer.apple.com still pulls from www.apple.com and images.apple.com, but a
+  separate brand domain or an unrelated third party (an embedded tracker, an
+  off-topic CDN) is left on the live web rather than mirrored. `--all-asset-hosts`
+  restores downloading assets from any host.
+- The progress line now counts real pages. "pages" is the number of distinct URL
+  paths written, and the query-string variants that one path can spawn by the
+  thousand are shown separately as "variants", so the live counter tracks the
+  site's real size instead of being inflated by `?q=…` permutations.
+
+### Fixed
+
+- An asset larger than the size cap (`--max-asset-mb`, 25 by default) is now
+  skipped instead of being truncated to a corrupt fragment. The cap was a
+  `LimitReader`, so an over-size file was saved as exactly the first N MB of
+  itself: a broken video or installer that wasted disk and would never play or
+  run. kage now checks the response size and leaves an over-cap asset out of the
+  mirror entirely, pointing at its live URL. On a developer.apple.com crawl this
+  was around a gigabyte of truncated WWDC videos and `.dmg` installers.
+- An asset URL whose query string carries a raw space is now requested with the
+  space percent-encoded, so the server gets a valid request instead of rejecting
+  it. Real sites bust a stylesheet's cache with a date, producing an href like
+  `styles/main.css?Thursday, 26-Feb-2026 16:26:41 UTC`; a browser encodes the
+  spaces before requesting, but kage was passing them through verbatim and the
+  server answered `400 Bad Request`. On a developer.apple.com crawl this was the
+  cause of the large majority of the download errors. The query is re-encoded on
+  the canonical URL, so the on-disk key matches the fixed request.
+
+## [0.2.1] - 2026-06-15
+
+### Added
+
+- ZIM archives now carry the metadata Kiwix and `zimcheck` treat as mandatory. Every archive gets a `Name` and a `Description` (a host-derived line when `--description` is not given), and, when the mirror has a usable favicon, an `Illustrator_48x48@1` entry: the icon rescaled to a 48x48 PNG, which is the book icon Kiwix shows in its library.
+
+## [0.2.0] - 2026-06-15
+
+### Added
+
+- `kage pack --app` wraps the packed viewer in a double-click desktop app with
+  the site's favicon as the icon. The flag builds on the binary format, so it
+  composes with `--base` (including a `webview` base) and `--icon`. On macOS it
+  writes a `.app` bundle (`Info.plist`, the viewer under `Contents/MacOS`, and an
+  `.icns` generated from the favicon); on Linux, with a Linux `--base`, it writes
+  an AppImage-style `.AppDir` and folds it into a single `.AppImage` when
+  `appimagetool` is installed. The icon is found in the mirror automatically
+  (preferring a large `apple-touch-icon.png`, then `favicon.png` or a PNG-based
+  `favicon.ico`) and can be overridden with `--icon`.
+- The release now ships a GUI-subsystem Windows base,
+  `kage_<version>_windows-gui_<arch>.zip`. Packing a viewer onto it with
+  `--format binary --base` produces a `.exe` that opens with no console window
+  behind it, the Windows equivalent of the `.app` double-click experience.
+
+### Changed
+
+- Cross-platform packing detects the base binary's target OS from its executable
+  header (ELF, PE, or Mach-O) rather than its file name, so a Windows viewer
+  always gets a `.exe` suffix and the run hint names the right platform even when
+  the base is named without one.
+
+## [0.1.2] - 2026-06-15
+
+### Security
+
+- Chrome now keeps its sandbox on by default. It was previously launched with `--no-sandbox` unconditionally, which removed Chrome's main line of defense when rendering pages from the open web (reported in #10). The sandbox is now dropped only where it genuinely cannot run: inside a container, or when running as root, and the choice is logged so it is never silent.
+
+### Added
+
+- Container-aware Chrome flags. kage detects a container from the `IN_DOCKER` environment variable or a `/.dockerenv` marker and, only there, drops the sandbox and adds `--disable-dev-shm-usage` (the default 64 MB `/dev/shm` is too small for Chrome on large pages). Outside a container the faster shared memory is left in place.
+- Asset downloads retry on a transient failure (a 403/429, a 5xx, or a network blip) with a short backoff, recovering files that bot-protection rejects on the first request of a burst. Permanent failures (404, 401, ...) are not retried.
+
+### Changed
+
+- Clearer crawl error reporting. Each failure is logged with a classified reason (`HTTP 403 Forbidden`, `timed out`, ...), the URL, and the page that referenced it, and the end-of-run summary lists what went wrong instead of printing only a count.
+
+### Fixed
+
+- The container image now runs. Chrome aborted on launch with `chrome_crashpad_handler: --database is required`, so kage disables Chrome's crash reporter inside a container, and the `kage` user now has a writable home (the mounted `/out` volume) so the default output, resume state, and Chrome's profile no longer fail with a permission error (issue #7).
+
+## [0.1.1] - 2026-06-14
+
+### Added
+
+- `kage pack <mirror-dir>` packs a cloned folder into one distributable file.
+  `--format zim` (the default) writes an open ZIM archive, the same single-file
+  format Kiwix uses; `--format binary` appends that archive to a copy of kage to
+  produce a self-contained executable that serves the site offline when run.
+  Flags cover the output path, metadata (`--title`, `--description`,
+  `--language`, `--date`), a `--base` binary for cross-platform viewers, and
+  `--no-compress`.
+- `kage open <file.zim>` serves a packed ZIM over a local HTTP server and opens
+  your browser, the read side of `kage pack --format zim`.
+- An optional native-window viewer. Built with `-tags webview` (which needs
+  cgo), `kage open` and a packed binary present the offline site in a real
+  window backed by the operating system's WebView (WKWebView, WebView2,
+  WebKitGTK) instead of a browser tab, so a packed kage feels like a standalone
+  app. The default build stays pure Go (`CGO_ENABLED=0`) and falls back to the
+  system browser, so the release pipeline is unchanged.
+- A pure-Go `zim` package that writes and reads the ZIM format: a fixed header,
+  MIME and pointer lists, zstd-compressed (or stored) clusters, redirects, and a
+  trailing MD5. It reads xz clusters so archives from other tooling open, and
+  writes zstd or stored only. Packing is deterministic: the same mirror produces
+  a byte-identical archive, with the UUID derived from the content rather than
+  randomised.
+
 ## [0.1.0] - 2026-06-14
 
 The first release. kage clones a live website into a self-contained folder you
@@ -39,5 +213,13 @@ can browse offline, with every script stripped out.
   a multi-arch container image on GHCR (Chromium bundled), checksums, SBOMs, and
   a cosign signature, all cut from one version tag by GoReleaser.
 
-[Unreleased]: https://github.com/tamnd/kage/compare/v0.1.0...HEAD
+[Unreleased]: https://github.com/tamnd/kage/compare/v0.3.3...HEAD
+[0.3.3]: https://github.com/tamnd/kage/compare/v0.3.2...v0.3.3
+[0.3.2]: https://github.com/tamnd/kage/compare/v0.3.1...v0.3.2
+[0.3.1]: https://github.com/tamnd/kage/compare/v0.3.0...v0.3.1
+[0.3.0]: https://github.com/tamnd/kage/compare/v0.2.1...v0.3.0
+[0.2.1]: https://github.com/tamnd/kage/compare/v0.2.0...v0.2.1
+[0.2.0]: https://github.com/tamnd/kage/compare/v0.1.2...v0.2.0
+[0.1.2]: https://github.com/tamnd/kage/compare/v0.1.1...v0.1.2
+[0.1.1]: https://github.com/tamnd/kage/compare/v0.1.0...v0.1.1
 [0.1.0]: https://github.com/tamnd/kage/releases/tag/v0.1.0

Dockerfile

@@ -29,7 +29,14 @@ WORKDIR /out
 # Point kage at the bundled Chromium and write mirrors under /out by default:
 #
 #   docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com
-ENV KAGE_CHROME=/usr/bin/chromium-browser
+#
+# The kage user has no home directory of its own, so HOME points at the mounted
+# /out volume. That keeps two things writable: kage's default output and resume
+# state (it lands under $HOME/data/kage), and Chrome's profile and crash
+# database. Without this both fail with a permission error in the container
+# (issue #7), and the mounted volume captures nothing.
+ENV KAGE_CHROME=/usr/bin/chromium-browser \
+    HOME=/out
 
 VOLUME ["/out"]
 

Makefile

@@ -10,11 +10,17 @@ LDFLAGS := -s -w \
 
 export CGO_ENABLED := 0
 
-.PHONY: build install test test-short vet tidy clean run
+.PHONY: build build-webview install test test-short vet tidy clean run
 
 build:
 	go build -ldflags "$(LDFLAGS)" -o bin/$(BIN) $(PKG)
 
+# A native-window viewer: opens packed sites in their own OS WebView window
+# instead of the browser. Needs cgo, so it is built separately from the default
+# pure-Go binary and the release pipeline.
+build-webview:
+	CGO_ENABLED=1 go build -tags webview -ldflags "$(LDFLAGS)" -o bin/$(BIN) $(PKG)
+
 install:
 	go install -ldflags "$(LDFLAGS)" $(PKG)
 

README.md

@@ -1,116 +1,203 @@
 # kage
 
-**kage** (影, "shadow") clones a website into a self-contained folder you can
-browse offline, with all the JavaScript stripped out. It renders every page in
-headless Chrome, snapshots the final rendered DOM, removes every script and
-event handler, and downloads the CSS, images, and fonts and rewrites them to
-local paths. The result looks like the live site but runs no code: a plain
-folder of `.html` files you can open straight from disk.
+[![ci](https://github.com/tamnd/kage/actions/workflows/ci.yml/badge.svg)](https://github.com/tamnd/kage/actions/workflows/ci.yml)
+[![Release](https://img.shields.io/github/v/release/tamnd/kage)](https://github.com/tamnd/kage/releases/latest)
+[![Go Reference](https://pkg.go.dev/badge/github.com/tamnd/kage.svg)](https://pkg.go.dev/github.com/tamnd/kage)
+[![Go Report Card](https://goreportcard.com/badge/github.com/tamnd/kage)](https://goreportcard.com/report/github.com/tamnd/kage)
+[![License](https://img.shields.io/github/license/tamnd/kage)](./LICENSE)
 
-```bash
-kage clone example.com
-kage serve kage-out/example.com
-```
+**kage** (影, "shadow") clones a website into a folder you can browse offline, with every script stripped out. It opens each page in real headless Chrome, waits for the page to settle, snapshots the DOM a human would have seen, then deletes all the JavaScript and pulls the CSS, images, and fonts down to local paths. What lands on disk looks like the live site and runs no code.
 
-## Why
+[Install](#install) • [Quick start](#quick-start) • [Commands](#commands) • [Clone](#clone) • [Pack](#pack-it-into-one-file) • [Double-click app](#a-double-click-app) • [Native window](#a-real-window-not-a-browser-tab) • [How it works](#how-it-works)
 
-Saving a page with "Save As" gives you a copy that still phones home, still runs
-analytics, and often renders blank because the markup is built by JavaScript at
-runtime. kage takes the opposite approach:
+![kage cloning paulgraham.com, packing it into one file, and serving it back offline](docs/static/demo.gif)
 
-- **Render first, save second.** Each page goes through real headless Chrome, so
-  a page whose content is assembled by JavaScript is captured the way a human
-  would have seen it, not as an empty shell.
-- **Strip every script.** Once the DOM is captured, kage removes all `<script>`
-  tags, every `on*` event handler, and any `javascript:` URL. The saved page is
-  inert: no tracking, no network calls, no surprises.
-- **Keep the layout.** Stylesheets, images, fonts, and media are downloaded and
-  rewritten to relative local paths, so the offline copy looks like the original.
-- **Stay browsable.** In-scope links are rewritten to point at the other saved
-  pages, so you can click around the mirror exactly as you would the live site.
+You already know the problem. You hit "Save As" on a page you want to keep, and six months later you open it to find a blank screen, a spinner that never stops, or a copy that still tries to phone home to an analytics server that no longer exists. The page was never really yours. It was a thin client for someone else's JavaScript.
+
+kage takes the other road. It drives a real browser, lets the page finish doing whatever it does, grabs the finished result, and then rips every script out of it. No tracking, no network calls, no surprises. Just `.html` files you can open straight off disk, hand to a friend, or pack into a single file and forget about for a decade.
+
+Full docs and guides live at **[kage.tamnd.com](https://kage.tamnd.com)**.
 
 ## Install
 
 ```bash
-# Go
 go install github.com/tamnd/kage/cmd/kage@latest
-
-# Homebrew (once the tap is published)
-brew install tamnd/tap/kage
-
-# Container (Chromium bundled)
-docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com
 ```
 
-Prebuilt archives, `.deb`/`.rpm`/`.apk` packages, and a multi-arch image are
-attached to each [release](https://github.com/tamnd/kage/releases).
-
-kage drives a real browser, so it needs Chrome or Chromium available. It finds a
-system install automatically; point it at a specific binary with `--chrome` or
-the `KAGE_CHROME` environment variable. The container image bundles Chromium.
-
-## Usage
+Prefer a prebuilt binary? Grab an archive, a `.deb`/`.rpm`/`.apk`, or a checksum from [releases](https://github.com/tamnd/kage/releases). Or skip installing Chrome yourself and use the container image, which bundles Chromium:
 
 ```bash
-kage clone <url> [flags]
-kage serve [dir] [flags]
+docker run --rm -v "$PWD/out:/out" ghcr.io/tamnd/kage clone paulgraham.com
 ```
 
-### Clone
+kage drives a real browser, so it needs Chrome or Chromium on the host. It finds a system install on its own; point it somewhere specific with `--chrome` or the `KAGE_CHROME` environment variable. The container needs nothing extra.
+
+Shell completion ships in the box: `kage completion bash|zsh|fish|powershell`.
+
+## Quick start
+
+Let's mirror Paul Graham's essays so you can read them on a plane, on a laptop with no wifi, or in the year 2050 after the site has finally changed its design:
 
 ```bash
-# Clone a whole site into kage-out/<host>/
-kage clone https://example.com
+# 1. Clone the site into $HOME/data/kage/paulgraham.com/
+kage clone paulgraham.com
 
-# Limit the crawl
-kage clone example.com --max-pages 200 --max-depth 3
+# 2. Read it back offline in your browser
+kage serve $HOME/data/kage/paulgraham.com
+# open http://127.0.0.1:8800
+```
 
-# Only a section of the site
-kage clone example.com --scope-prefix /docs
+That's the whole loop. Every essay, every image, every stylesheet, frozen on your disk and runnable with zero network. The next two steps are optional but nice: collapse the whole thing into one file, and pop it open in its own window.
 
-# Include subdomains, and trigger lazy-loaded images by scrolling
+```bash
+# 3. Squeeze the mirror into a single shareable file
+kage pack paulgraham.com               # -> paulgraham.com.zim
+kage open paulgraham.com.zim
+
+# 4. Or into one executable that *is* the site
+kage pack paulgraham.com --format binary -o paulgraham
+./paulgraham                           # serves itself, needs nothing installed
+```
+
+## Commands
+
+| Command | What it does |
+| --- | --- |
+| `kage clone <url>` | render a site in headless Chrome and write a browsable, script-free mirror |
+| `kage serve [dir]` | preview a cloned folder over a local HTTP server |
+| `kage pack <mirror-dir>` | collapse a mirror into one ZIM archive, a self-contained viewer binary, or a double-click app |
+| `kage open <file.zim>` | serve a packed ZIM back for offline reading |
+
+## Clone
+
+```bash
+# The whole site, into $HOME/data/kage/<host>/
+kage clone https://paulgraham.com
+
+# Just the first 50 pages, two links deep, for a quick taste
+kage clone paulgraham.com --max-pages 50 --max-depth 2
+
+# Only one section of a bigger site
+kage clone go.dev --scope-prefix /doc
+
+# Pull in subdomains too, and scroll each page to trip lazy-loaded images
 kage clone example.com --subdomains --scroll
 
-# Resume an interrupted run (on by default; Ctrl-C saves state)
-kage clone example.com
-
-# Re-render every page in place to pull in changed content
-kage clone example.com --refresh
+# Come back next month and re-render in place to catch new essays
+kage clone paulgraham.com --refresh
 ```
 
-A clone is idempotent: each page is keyed by the file it writes, so the same URL
-reached over http and https, with or without a trailing slash, is fetched once.
-Re-running resumes where it left off; `--refresh` re-renders in place, `--force`
-wipes and starts clean.
+A clone is a polite, breadth-first crawl. It reads `robots.txt`, seeds itself from `sitemap.xml`, and stays on the seed host unless you tell it otherwise. It is also stubbornly idempotent: each page is keyed by the file it writes, so the same essay reached over http and https, with or without a trailing slash, gets fetched exactly once. Hit Ctrl-C and it saves its place on the way out; run it again and it picks up where it stopped. `--refresh` re-renders in place, `--force` wipes the host and starts clean.
 
-Common flags:
+The flags you'll actually reach for:
 
 | Flag | Default | Meaning |
 |------|---------|---------|
 | `-o, --out` | `$HOME/data/kage` | Output root; the mirror lands in `<out>/<host>/` |
-| `-p, --max-pages` | `0` | Stop after N pages (0 = unlimited) |
-| `-d, --max-depth` | `0` | Link-follow depth cap (0 = unlimited) |
-| `--scope-prefix` | | Only crawl pages whose path starts with this prefix |
+| `-p, --max-pages` | `0` | Stop after N pages (0 = no limit) |
+| `-d, --max-depth` | `0` | How many links deep to follow (0 = no limit) |
+| `--scope-prefix` | | Only crawl paths starting with this prefix |
 | `--subdomains` | `false` | Treat subdomains of the seed host as in scope |
 | `--exclude` | | Path prefixes to skip (repeatable) |
 | `--scroll` | `false` | Auto-scroll each page to trigger lazy loading |
-| `--workers` | `4` | Concurrent page render workers |
-| `--no-robots` | `false` | Ignore `robots.txt` (be polite) |
+| `--workers` | `4` | How many pages to render at once |
+| `--no-robots` | `false` | Ignore `robots.txt` (be nice) |
 | `-f, --force` | `false` | Delete any existing mirror for the host first |
 | `--chrome` | | Path to the Chrome/Chromium binary |
 
-Run `kage clone --help` for the full list.
+`kage clone --help` has the rest, including render-timing, concurrency, and asset-size knobs.
 
 ### Serve
 
-`kage serve` runs a local static file server over a cloned folder so links and
-assets resolve the way they would on a real host:
+`kage serve` runs a tiny static file server over a cloned folder so links and assets resolve the way they would on a real host:
 
 ```bash
-kage serve kage-out/example.com
+kage serve $HOME/data/kage/paulgraham.com
 # open http://127.0.0.1:8800
 ```
 
+## Pack it into one file
+
+A mirror is a folder, which is great for browsing and lousy for moving around. Copying thousands of little files is slow, and "here, have this directory" is a clumsy thing to hand someone. `kage pack` collapses the whole mirror into one artifact, and you choose the shape: an open ZIM archive, or a single executable that *is* the site.
+
+### A single ZIM file
+
+```bash
+kage pack paulgraham.com               # -> paulgraham.com.zim
+kage open paulgraham.com.zim
+```
+
+ZIM is an open file format built for exactly this: a whole website (or a whole Wikipedia) squeezed into one compressed, indexed, read-only file. kage writes the entire mirror into it, text zstd-compressed and media stored as-is. It is the format behind [Kiwix](https://kiwix.org), the offline-content project people use to carry Wikipedia, Stack Overflow, and Project Gutenberg onto boats, into classrooms with no internet, and onto a phone for a long flight. Because the format is a documented standard and not a kage invention, a `paulgraham.com.zim` you make today will still open in any ZIM reader years from now.
+
+So you are not locked into kage. `kage open` is the quickest way back in, but the very same file works across the wider Kiwix ecosystem:
+
+```bash
+kage open paulgraham.com.zim            # read it back with kage
+kiwix-serve paulgraham.com.zim          # or serve it with Kiwix at http://localhost
+```
+
+You can also double-click the file in the [Kiwix desktop app](https://kiwix.org/en/applications/), or load it on [Kiwix for Android or iOS](https://kiwix.org/en/applications/) to read your mirror on your phone. One caveat: kage writes a structurally valid archive with the standard metadata, but it does not build the full-text search index that Kiwix's own packs ship with, so browsing and clicking work everywhere while in-reader search is limited.
+
+Packing is deterministic. The same mirror always produces a byte-identical file, with the archive UUID derived from the content instead of randomized, so a pack is safe to checksum and cache. A bare host name resolves against the default output directory, which is why `kage pack paulgraham.com` just works right after `kage clone paulgraham.com`.
+
+### A self-contained binary
+
+`--format binary` glues the archive onto a copy of kage and hands you a single executable that serves the site offline when you run it. Whoever you send it to needs nothing installed: not kage, not a ZIM reader, nothing.
+
+```bash
+kage pack paulgraham.com --format binary -o paulgraham
+./paulgraham
+```
+
+The appended archive is platform-independent; only the base executable carries the architecture. By default kage appends to itself, so you get a viewer for the machine you ran it on. Point `--base` at a kage built for another OS (grab one from a [release](https://github.com/tamnd/kage/releases); every platform ships one) to produce a viewer for that platform from your own machine. kage reads the base's executable header to figure out the target, so a Windows viewer automatically gets a `.exe` name:
+
+```bash
+# Sitting on a Mac, build a Windows viewer
+kage pack paulgraham.com --format binary --base kage-windows-amd64.exe   # -> paulgraham.exe
+```
+
+The trade is size. The binary carries a whole kage, so it weighs around 13 MiB plus the site no matter how small the mirror is. When you only need the content, the ZIM is far leaner.
+
+### A double-click app
+
+A bare binary is great from a terminal, but double-click it in a file manager and the experience is rough: macOS opens a Terminal window behind the site, and on Windows a console flashes up next to it. Add `--app` and kage wraps the same viewer in a proper desktop app so a double-click just opens the site, no terminal, with the mirror's own favicon as the icon.
+
+On macOS you get a real `.app` bundle:
+
+```bash
+kage pack paulgraham.com --app                 # -> paulgraham.app
+open paulgraham.app                            # or double-click it in Finder
+```
+
+On Linux, point `--base` at a Linux kage and you get an [AppImage](https://appimage.org)-style `.AppDir` with a `.desktop` launcher (`Terminal=false`, so no console). If [`appimagetool`](https://github.com/AppImage/appimagetool) is installed, kage folds it into a single double-clickable `.AppImage` for you:
+
+```bash
+kage pack paulgraham.com --app --base kage-linux-amd64   # -> paulgraham.AppDir (+ .AppImage)
+```
+
+kage finds the icon by digging the favicon out of the mirror (it prefers a large `apple-touch-icon.png` and falls back to `favicon.ico`); pass `--icon some.png` to override it. Pair `--app` with a `webview` base (below) and the double-click opens a native window instead of the browser, which is the full "it's an app" effect.
+
+Windows needs no bundle, because there a single `.exe` already is the app. The catch is the console window. The release ships a `kage_<version>_windows-gui_<arch>.zip` whose binary is linked for the GUI subsystem, so a viewer packed onto it opens with no console behind it:
+
+```bash
+# Build a console-free Windows viewer (from any OS)
+kage pack paulgraham.com --format binary --base kage-windows-gui-amd64.exe   # -> paulgraham.exe
+```
+
+## A real window, not a browser tab
+
+By default a packed binary opens your system browser, which means the site shows up as yet another tab, address bar and all, next to the 47 you already have open. Build kage with the `webview` tag and it opens the site in its own window instead, backed by the operating system's WebView (WKWebView on macOS, WebView2 on Windows, WebKitGTK on Linux). Paul Graham's essays, offline, in something that looks and feels like a real app:
+
+![paulgraham.com served offline in a native kage window](docs/static/webview.png)
+
+```bash
+make build-webview                       # or: CGO_ENABLED=1 go build -tags webview ./cmd/kage
+kage pack paulgraham.com --format binary --base bin/kage -o paulgraham
+./paulgraham                             # opens a window, no browser in sight
+```
+
+This build needs cgo and links the platform WebView, so it stays opt-in. The default build is pure Go (`CGO_ENABLED=0`) and the prebuilt release binaries open the browser, which keeps the cross-compiled release simple. `kage open` honours the same tag, so built with `-tags webview` it shows a ZIM in a native window too.
+
 ## How it works
 
 ```
@@ -118,33 +205,59 @@ seed URL ─▶ headless Chrome ─▶ final DOM ─▶ strip JS ─▶ localise
               (render)          (snapshot)   (sanitize)   (rewrite links)
 ```
 
-A clone is a polite breadth-first crawl. Pages are rendered by a pool of Chrome
-tabs; assets are fetched over plain HTTP by a separate worker pool. Every URL
-maps deterministically to a local path, so links can be rewritten before the
-asset they point at has even finished downloading. The crawl honours
-`robots.txt` and seeds itself from `sitemap.xml` by default. Output layout:
+A pool of Chrome tabs renders pages; a separate pool fetches assets over plain HTTP. Every URL maps deterministically to a local path, so links get rewritten before the asset they point at has even finished downloading. The output looks like this:
 
 ```
-kage-out/example.com/
-├── index.html                 # the home page, scripts stripped
-├── about/index.html           # /about
+paulgraham.com/
+├── index.html                  # the home page, scripts stripped
+├── greatwork.html              # /greatwork.html, an essay
 ├── _kage/                      # reserved: assets and crawl state
-│   ├── example.com/site.css    # localised stylesheet (url() rewritten)
-│   ├── example.com/logo.png
-│   └── state.json              # visited set, for --resume
+│   ├── paulgraham.com/site.css  # localised stylesheet (url() rewritten)
+│   ├── paulgraham.com/pg.png
+│   └── state.json              # visited set, for resuming
 └── ...
 ```
 
+`pack` rides on the same idea: the mirror's links are already mirror-relative paths, and those map one-to-one onto the archive's content entries, so a click in a served page hits the right entry with no rewriting at all.
+
 ## Building from source
 
 ```bash
 git clone https://github.com/tamnd/kage
 cd kage
-make build          # -> bin/kage
-make test           # full suite, including Chrome-driven end-to-end tests
+make build          # -> bin/kage (pure Go, opens the browser)
+make build-webview  # -> bin/kage with the native-window viewer (needs cgo)
+make test           # full suite, including the Chrome-driven end-to-end tests
 make test-short     # skip the tests that launch a browser
 ```
 
+The repo is split by concern:
+
+```
+cmd/kage/   thin main: pins the main thread, then hands off to cli.Execute
+cli/        the cobra command tree and flag wiring
+clone/      the crawl: frontier, render workers, asset workers, resume state
+browser/    headless Chrome control and DOM snapshotting
+sanitize/   strip scripts, handlers, and javascript: URLs from the DOM
+asset/      download and localise CSS, images, and fonts
+urlx/       the deterministic URL-to-path mapping
+zim/        a pure-Go ZIM reader and writer
+pack/       mirror to ZIM or self-contained binary, and the offline HTTP handler
+viewer/     present a served site: system browser, or native window (webview tag)
+docs/       the tago documentation site
+```
+
+## Releasing
+
+Push a version tag and GitHub Actions runs GoReleaser, which builds the archives, the `.deb`/`.rpm`/`.apk` packages, a multi-arch GHCR image with Chromium bundled, checksums, SBOMs, and a cosign signature:
+
+```bash
+git tag v0.1.1
+git push --tags
+```
+
+The image tag carries no `v` prefix (`ghcr.io/tamnd/kage:0.1.1`). The Homebrew and Scoop steps self-disable until their tokens exist, so the first release works with no extra secrets.
+
 ## License
 
 MIT. See [LICENSE](LICENSE).

asset/asset_test.go

@@ -69,6 +69,7 @@ body { background: url(../img/bg.png); }
 
 const htmlDoc = `<!doctype html><html><head>
 <link rel="stylesheet" href="/css/main.css">
+<link rel="preload stylesheet" href="/css/vp.css" as="style">
 <link rel="icon" href="/favicon.ico">
 <link rel="canonical" href="https://ex.com/canon">
 </head><body>
@@ -98,6 +99,7 @@ func TestRewriteHTML(t *testing.T) {
 
 	checks := map[string]bool{
 		`href="_kage/ex.com/css/main.css"`:                  true, // stylesheet localised
+		`href="_kage/ex.com/css/vp.css"`:                    true, // multi-value "preload stylesheet" rel localised
 		`href="_kage/ex.com/favicon.ico"`:                   true, // icon localised
 		`href="https://ex.com/canon"`:                       true, // canonical left alone
 		`href="docs/intro/index.html"`:                      true, // internal page → local

asset/download.go

@@ -2,6 +2,7 @@ package asset
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -18,6 +19,7 @@ type Downloader struct {
 	Client    *http.Client
 	UserAgent string
 	MaxBytes  int64 // per-asset cap; 0 = unlimited
+	Retries   int   // extra attempts for a transient failure (0 = try once)
 }
 
 // NewDownloader builds a Downloader with a sane client and the given timeout.
@@ -26,6 +28,10 @@ func NewDownloader(userAgent string, timeout time.Duration, maxBytes int64) *Dow
 		Client:    &http.Client{Timeout: timeout},
 		UserAgent: userAgent,
 		MaxBytes:  maxBytes,
+		// A few sites (and the bot-protection in front of them) reject the first
+		// request of a burst with a 403 or 429 but serve a retry fine, so give
+		// transient failures a couple of extra tries before giving up.
+		Retries: 3,
 	}
 }
 
@@ -36,9 +42,62 @@ type Result struct {
 	IsCSS       bool
 }
 
+// ErrTooLarge reports that an asset exceeds the size cap and was skipped without
+// being saved. It is deliberately a skip, not a download failure: the caller
+// leaves the asset out of the mirror rather than writing a truncated fragment of
+// it, so a 500 MB installer or video never bloats the archive with a corrupt
+// quarter of itself.
+var ErrTooLarge = errors.New("asset over size cap")
+
+// StatusError reports a non-2xx HTTP response. It carries the code so callers
+// can render a clear message ("HTTP 403 Forbidden") and decide whether a retry
+// is worthwhile, without the URL baked in (the caller already has it).
+type StatusError struct {
+	Code int
+}
+
+func (e *StatusError) Error() string {
+	if t := http.StatusText(e.Code); t != "" {
+		return fmt.Sprintf("HTTP %d %s", e.Code, t)
+	}
+	return fmt.Sprintf("HTTP %d", e.Code)
+}
+
 // Get fetches u, sending referer as the Referer header. It reads at most
 // MaxBytes and reports whether the body is CSS (so the caller can rewrite it).
+// A transient failure (a 403/429/5xx or a network blip) is retried with a short
+// backoff up to Retries times.
 func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Result, error) {
+	attempts := d.Retries + 1
+	if attempts < 1 {
+		attempts = 1
+	}
+	var lastErr error
+	for i := 0; i < attempts; i++ {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		if i > 0 {
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			case <-time.After(backoff(i)):
+			}
+		}
+		res, err := d.try(ctx, u, referer)
+		if err == nil {
+			return res, nil
+		}
+		lastErr = err
+		if !transient(err) {
+			break
+		}
+	}
+	return nil, lastErr
+}
+
+// try performs a single fetch attempt.
+func (d *Downloader) try(ctx context.Context, u *url.URL, referer string) (*Result, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
 	if err != nil {
 		return nil, err
@@ -55,16 +114,28 @@ func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Resu
 	}
 	defer func() { _ = resp.Body.Close() }()
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return nil, fmt.Errorf("status %d for %s", resp.StatusCode, u)
+		return nil, &StatusError{Code: resp.StatusCode}
+	}
+	// Skip an over-cap asset instead of truncating it. A Content-Length lets us
+	// bail before reading a byte; otherwise we read one byte past the cap and, if
+	// the body really is larger, discard what we have. Either way nothing partial
+	// reaches disk.
+	if d.MaxBytes > 0 && resp.ContentLength > d.MaxBytes {
+		return nil, ErrTooLarge
 	}
 	var r io.Reader = resp.Body
 	if d.MaxBytes > 0 {
-		r = io.LimitReader(resp.Body, d.MaxBytes)
+		// Read at most one byte past the cap so a body with no (or a lying)
+		// Content-Length cannot stream gigabytes into memory before we notice.
+		r = io.LimitReader(resp.Body, d.MaxBytes+1)
 	}
 	body, err := io.ReadAll(r)
 	if err != nil {
 		return nil, err
 	}
+	if d.MaxBytes > 0 && int64(len(body)) > d.MaxBytes {
+		return nil, ErrTooLarge
+	}
 	ct := resp.Header.Get("Content-Type")
 	return &Result{
 		Body:        body,
@@ -73,6 +144,37 @@ func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Resu
 	}, nil
 }
 
+// backoff returns the pause before retry attempt i (1-based): 500ms, 1s, 2s, …
+func backoff(i int) time.Duration {
+	d := 500 * time.Millisecond << (i - 1)
+	if max := 5 * time.Second; d > max {
+		d = max
+	}
+	return d
+}
+
+// transient reports whether an error is worth retrying. Bot-protection statuses
+// (403/429), request-timeout and too-early (408/425), and 5xx server errors are
+// transient; other 4xx (404, 401, 410, …) are permanent. A network error is
+// retried, but a cancelled or expired context is not.
+func transient(err error) bool {
+	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+		return false
+	}
+	if errors.Is(err, ErrTooLarge) {
+		return false
+	}
+	var se *StatusError
+	if errors.As(err, &se) {
+		switch se.Code {
+		case http.StatusForbidden, http.StatusRequestTimeout, http.StatusTooEarly, http.StatusTooManyRequests:
+			return true
+		}
+		return se.Code >= 500
+	}
+	return true
+}
+
 // isCSS reports whether a response is a stylesheet, by content-type or by a
 // .css path when the server sends no useful type.
 func isCSS(contentType string, u *url.URL) bool {

asset/download_test.go

@@ -0,0 +1,175 @@
+package asset
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestStatusErrorMessage(t *testing.T) {
+	cases := map[int]string{
+		403: "HTTP 403 Forbidden",
+		404: "HTTP 404 Not Found",
+		999: "HTTP 999",
+	}
+	for code, want := range cases {
+		if got := (&StatusError{Code: code}).Error(); got != want {
+			t.Errorf("StatusError{%d} = %q; want %q", code, got, want)
+		}
+	}
+}
+
+func TestGetRetriesTransientThenSucceeds(t *testing.T) {
+	var hits int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// 403 on the first try (like bot-protection), then serve the file.
+		if atomic.AddInt32(&hits, 1) == 1 {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		w.Header().Set("Content-Type", "text/css")
+		_, _ = w.Write([]byte("body{}"))
+	}))
+	defer srv.Close()
+
+	d := NewDownloader("kage-test", 5*time.Second, 0)
+	u, _ := url.Parse(srv.URL + "/style.css")
+	res, err := d.Get(context.Background(), u, "")
+	if err != nil {
+		t.Fatalf("Get after retry: %v", err)
+	}
+	if !res.IsCSS || string(res.Body) != "body{}" {
+		t.Errorf("unexpected result: css=%v body=%q", res.IsCSS, res.Body)
+	}
+	if hits < 2 {
+		t.Errorf("expected a retry; server saw %d hits", hits)
+	}
+}
+
+func TestGetDoesNotRetryPermanent(t *testing.T) {
+	var hits int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hits, 1)
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv.Close()
+
+	d := NewDownloader("kage-test", 5*time.Second, 0)
+	u, _ := url.Parse(srv.URL + "/missing.png")
+	_, err := d.Get(context.Background(), u, "")
+
+	var se *StatusError
+	if !errors.As(err, &se) || se.Code != 404 {
+		t.Fatalf("got %v; want StatusError 404", err)
+	}
+	if hits != 1 {
+		t.Errorf("404 should not be retried; server saw %d hits", hits)
+	}
+}
+
+func TestGetGivesUpAfterRetries(t *testing.T) {
+	var hits int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hits, 1)
+		w.WriteHeader(http.StatusTooManyRequests)
+	}))
+	defer srv.Close()
+
+	d := NewDownloader("kage-test", 5*time.Second, 0)
+	d.Retries = 2
+	u, _ := url.Parse(srv.URL + "/rate.css")
+	_, err := d.Get(context.Background(), u, "")
+
+	var se *StatusError
+	if !errors.As(err, &se) || se.Code != 429 {
+		t.Fatalf("got %v; want StatusError 429", err)
+	}
+	if hits != 3 { // 1 try + 2 retries
+		t.Errorf("expected 3 attempts, server saw %d", hits)
+	}
+}
+
+func TestGetSkipsOverCapByContentLength(t *testing.T) {
+	var hits int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hits, 1)
+		body := make([]byte, 4096) // declared via Content-Length
+		w.Header().Set("Content-Type", "video/mp4")
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	d := NewDownloader("kage-test", 5*time.Second, 1024) // cap below the body
+	u, _ := url.Parse(srv.URL + "/clip.mp4")
+	_, err := d.Get(context.Background(), u, "")
+	if !errors.Is(err, ErrTooLarge) {
+		t.Fatalf("got %v; want ErrTooLarge", err)
+	}
+	if hits != 1 {
+		t.Errorf("an over-cap asset should not be retried; server saw %d hits", hits)
+	}
+}
+
+func TestGetSkipsOverCapWithoutContentLength(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// A chunked response carries no Content-Length, so the cap is only known
+		// after reading past it.
+		w.Header().Set("Content-Type", "application/octet-stream")
+		fl, _ := w.(http.Flusher)
+		chunk := make([]byte, 512)
+		for i := 0; i < 8; i++ {
+			_, _ = w.Write(chunk)
+			if fl != nil {
+				fl.Flush()
+			}
+		}
+	}))
+	defer srv.Close()
+
+	d := NewDownloader("kage-test", 5*time.Second, 1024)
+	u, _ := url.Parse(srv.URL + "/stream.bin")
+	_, err := d.Get(context.Background(), u, "")
+	if !errors.Is(err, ErrTooLarge) {
+		t.Fatalf("got %v; want ErrTooLarge", err)
+	}
+}
+
+func TestGetKeepsUnderCap(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "image/png")
+		_, _ = w.Write([]byte("small"))
+	}))
+	defer srv.Close()
+
+	d := NewDownloader("kage-test", 5*time.Second, 1024)
+	u, _ := url.Parse(srv.URL + "/logo.png")
+	res, err := d.Get(context.Background(), u, "")
+	if err != nil {
+		t.Fatalf("under-cap asset: %v", err)
+	}
+	if string(res.Body) != "small" {
+		t.Errorf("body = %q; want %q", res.Body, "small")
+	}
+}
+
+func TestTransientClassification(t *testing.T) {
+	transientCodes := []int{403, 408, 425, 429, 500, 502, 503}
+	for _, c := range transientCodes {
+		if !transient(&StatusError{Code: c}) {
+			t.Errorf("status %d should be transient", c)
+		}
+	}
+	for _, c := range []int{400, 401, 404, 410} {
+		if transient(&StatusError{Code: c}) {
+			t.Errorf("status %d should be permanent", c)
+		}
+	}
+	if transient(context.Canceled) {
+		t.Error("context.Canceled should not be transient")
+	}
+}

asset/html.go

@@ -9,13 +9,28 @@ import (
 	"golang.org/x/net/html/atom"
 )
 
-// stylesheetRels are <link rel> values whose href kage downloads as an asset.
-var stylesheetRels = map[string]bool{
-	"stylesheet": true, "icon": true, "shortcut icon": true,
+// assetRels are the individual <link rel> tokens whose href kage downloads as
+// an asset. A rel attribute is a space-separated token list, so a single known
+// token in it (for example the "stylesheet" in "preload stylesheet") is enough.
+var assetRels = map[string]bool{
+	"stylesheet": true, "icon": true,
 	"apple-touch-icon": true, "apple-touch-icon-precomposed": true,
 	"mask-icon": true, "manifest": true, "preload": true, "prefetch": true,
 }
 
+// linkRelDownloadable reports whether a <link rel> names a resource kage should
+// download. It treats rel as the space-separated token list the HTML spec
+// defines, so "preload stylesheet", "shortcut icon", or a bare "stylesheet" all
+// match on a single recognised token.
+func linkRelDownloadable(rel string) bool {
+	for _, tok := range strings.Fields(strings.ToLower(rel)) {
+		if assetRels[tok] {
+			return true
+		}
+	}
+	return false
+}
+
 // RewriteHTML walks the parsed document and rewrites every resource and link
 // reference through sink, resolving relative URLs against base. It mutates the
 // tree in place; the caller renders it afterwards. References kage cannot handle
@@ -40,8 +55,7 @@ func rewriteElement(n *html.Node, base *url.URL, sink RefSink) {
 	case atom.Iframe, atom.Frame:
 		rewriteAttr(n, "src", base, sink, pageOrAsset)
 	case atom.Link:
-		rel := strings.ToLower(strings.TrimSpace(attrVal(n, "rel")))
-		if stylesheetRels[rel] {
+		if linkRelDownloadable(attrVal(n, "rel")) {
 			rewriteAttr(n, "href", base, sink, alwaysAsset)
 		}
 	case atom.Img:

browser/pool.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strings"
 	"sync"
 	"time"
 
@@ -66,6 +67,21 @@ type RenderResult struct {
 	Title    string
 }
 
+// ErrNotHTML reports that a URL kage tried to render as a page is not HTML: the
+// server returned some other content type (a zip, a CSV, a PDF, a bare image).
+// Such a URL reaches the page worker when its link carried no file extension to
+// classify it by. The caller reroutes it to the asset downloader, where the
+// asset policy decides whether to localise or leave it remote, instead of saving
+// an empty or broken page or letting Chrome download it (issue #32).
+type ErrNotHTML struct {
+	URL         string
+	ContentType string
+}
+
+func (e *ErrNotHTML) Error() string {
+	return fmt.Sprintf("not HTML (%s): %s", e.ContentType, e.URL)
+}
+
 // Render navigates to rawURL, lets it settle, and returns the final rendered
 // HTML. It acquires a page slot from the pool and releases it when done.
 func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error) {
@@ -89,8 +105,23 @@ func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error)
 
 	page = page.Context(ctx).Timeout(p.opts.RenderTimeout)
 
-	if err := page.Navigate(rawURL); err != nil {
-		return RenderResult{}, fmt.Errorf("navigate %s: %w", rawURL, err)
+	// Watch the main document's response so a navigation that turns out to be a
+	// non-HTML resource (a zip, a CSV, a bare image) is caught and handed back for
+	// the asset downloader, rather than rendered as a broken page or, with downloads
+	// denied, left as an aborted navigation (issue #32). The content type arrives in
+	// the response headers whether Chrome renders the body or aborts it as a denied
+	// download, so this catches both.
+	mainContentType := watchMainDocument(page)
+
+	navErr := page.Navigate(rawURL)
+	// A denied download aborts the navigation, so inspect the captured content type
+	// before treating a navigation error as a failure. waitFor gives the response
+	// event a brief moment to be processed; for an HTML page it returns at once.
+	if ct := waitFor(ctx, mainContentType, 2*time.Second); ct != "" && !isHTML(ct) {
+		return RenderResult{}, &ErrNotHTML{URL: rawURL, ContentType: ct}
+	}
+	if navErr != nil {
+		return RenderResult{}, fmt.Errorf("navigate %s: %w", rawURL, navErr)
 	}
 	if err := page.WaitLoad(); err != nil {
 		return RenderResult{}, fmt.Errorf("wait load %s: %w", rawURL, err)
@@ -130,9 +161,30 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
 		l := launcher.New().
 			Headless(p.opts.Headless).
 			Set("disable-blink-features", "AutomationControlled").
-			Set("disable-dev-shm-usage", "").
-			Set("no-sandbox", "").
 			Set("disable-gpu", "")
+
+		// Chrome's sandbox is the main line of defense when rendering pages from
+		// the open web, so kage keeps it on by default (issue #10). It is dropped
+		// only where it genuinely cannot initialize: inside a container, or when
+		// running as root, where Chrome otherwise refuses to start. The decision
+		// is logged so it is never silent.
+		if off, reason := disableSandbox(); off {
+			l = l.Set("no-sandbox", "")
+			warnSandboxDisabled(reason)
+		}
+
+		// In a container, the default /dev/shm is only 64 MB, too small for
+		// Chrome's renderer on large pages, so steer it to a temp file instead.
+		// Outside a container /dev/shm is roomy and faster, so leave it alone.
+		// Chrome's crashpad handler also aborts with "--database is required" in a
+		// minimal container, which fails the whole launch (issue #7), so turn the
+		// crash reporter off there. kage never uploads Chrome crash dumps anyway.
+		if inContainer() {
+			l = l.Set("disable-dev-shm-usage", "").
+				Set("disable-crash-reporter", "").
+				Set("disable-breakpad", "")
+		}
+
 		if bin := p.chromeBin(); bin != "" {
 			l = l.Bin(bin)
 		}
@@ -147,6 +199,21 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
 	if err := b.Connect(); err != nil {
 		return nil, fmt.Errorf("connect Chrome: %w", err)
 	}
+
+	// kage never wants Chrome to write a file to disk. Every asset is fetched
+	// through kage's own downloader, which applies the size and media policy, so a
+	// Chrome-initiated download is only ever an accident: navigating an <a> link
+	// that turns out to be a binary (a zip, an installer, a CSV) makes Chrome save
+	// it to the user's Downloads folder, a surprise side effect of a crawl
+	// (issue #32). Denying downloads browser-wide stops that. The navigation is
+	// aborted instead, and Render's non-HTML detection reroutes the URL through the
+	// asset downloader, where the asset policy decides its fate. This is
+	// best-effort: if the call is unsupported, the non-HTML detection still keeps
+	// the binary out of the saved mirror.
+	_ = proto.BrowserSetDownloadBehavior{
+		Behavior: proto.BrowserSetDownloadBehaviorBehaviorDeny,
+	}.Call(b)
+
 	p.browser = b
 	return b, nil
 }
@@ -225,6 +292,148 @@ func systemChromeCandidates() []string {
 	}
 }
 
+// disableSandbox decides whether Chrome should launch without its sandbox, with
+// a short reason for the log. The secure default is to keep the sandbox on; it
+// is dropped only where it cannot run: inside a container, or when running as
+// root (Chrome refuses to start a sandbox as root).
+func disableSandbox() (off bool, reason string) {
+	if inContainer() {
+		return true, "container"
+	}
+	if isRoot() {
+		return true, "root"
+	}
+	return false, ""
+}
+
+// warnSandboxDisabled prints why the sandbox was turned off, so dropping a
+// security boundary is always visible rather than silent.
+func warnSandboxDisabled(reason string) {
+	switch reason {
+	case "container":
+		fmt.Fprintln(os.Stderr, "kage: container detected, Chrome sandbox disabled")
+	case "root":
+		fmt.Fprintln(os.Stderr, "kage: running as root, Chrome sandbox disabled (run as a non-root user to keep it on)")
+	}
+}
+
+// inContainer reports whether kage is running inside a container, where Chrome
+// needs container-specific flags. It honors IN_DOCKER (set it in your image)
+// and the /.dockerenv marker that Docker writes into every container.
+//
+// Keeping the sandbox on by default and dropping it only here was prompted by
+// Dimitrios Prasakis (issue #10); the IN_DOCKER opt-in was suggested on Hacker
+// News (https://news.ycombinator.com/item?id=48534865). Thanks to both.
+func inContainer() bool {
+	if envTrue("IN_DOCKER") {
+		return true
+	}
+	if _, err := os.Stat("/.dockerenv"); err == nil {
+		return true
+	}
+	return false
+}
+
+// isRoot reports whether the process runs as the superuser. On Windows
+// os.Geteuid returns -1, so this is false there.
+func isRoot() bool {
+	return os.Geteuid() == 0
+}
+
+// envTrue reports whether the named environment variable is set to a truthy
+// value.
+func envTrue(name string) bool {
+	v, ok := envBool(name)
+	return ok && v
+}
+
+// envBool parses a boolean-ish environment variable. It returns ok=false when
+// the variable is unset or empty. "1", "true", "yes", "on" are true and "0",
+// "false", "no", "off" are false (case-insensitive); any other non-empty value
+// counts as true, so IN_DOCKER=docker reads as set.
+func envBool(name string) (val, ok bool) {
+	s := strings.TrimSpace(os.Getenv(name))
+	if s == "" {
+		return false, false
+	}
+	switch strings.ToLower(s) {
+	case "1", "true", "yes", "on":
+		return true, true
+	case "0", "false", "no", "off":
+		return false, true
+	default:
+		return true, true
+	}
+}
+
+// watchMainDocument subscribes to network responses and returns an accessor for
+// the main document's content type. The first Document-type response is the main
+// frame's navigation; later Document responses are sub-frames (iframes), whose
+// type kage does not police, so only the first is kept. The accessor is safe to
+// call from another goroutine. Any setup error leaves the accessor returning "",
+// which the caller reads as "unknown, render normally".
+func watchMainDocument(page *rod.Page) func() string {
+	var (
+		mu sync.Mutex
+		ct string
+	)
+	if err := (proto.NetworkEnable{}).Call(page); err != nil {
+		return func() string { return "" }
+	}
+	wait := page.EachEvent(func(e *proto.NetworkResponseReceived) {
+		if e.Type != proto.NetworkResourceTypeDocument || e.Response == nil {
+			return
+		}
+		mu.Lock()
+		if ct == "" {
+			ct = e.Response.MIMEType
+		}
+		mu.Unlock()
+	})
+	// EachEvent's wait blocks until the page context ends, draining events as they
+	// arrive; run it for the page's lifetime. The deferred page.Close in Render
+	// cancels the context and unblocks it.
+	go wait()
+	return func() string {
+		mu.Lock()
+		defer mu.Unlock()
+		return ct
+	}
+}
+
+// waitFor polls get until it returns a non-empty value, the deadline passes, or
+// the context is cancelled, then returns whatever it last saw. It exists because
+// the network response is processed on another goroutine, so the value may not be
+// set the instant Navigate returns; an HTML page sets it within a few
+// milliseconds, while a never-arriving response simply waits out the deadline.
+func waitFor(ctx context.Context, get func() string, deadline time.Duration) string {
+	const step = 20 * time.Millisecond
+	for waited := time.Duration(0); waited < deadline; waited += step {
+		if v := get(); v != "" {
+			return v
+		}
+		select {
+		case <-ctx.Done():
+			return get()
+		case <-time.After(step):
+		}
+	}
+	return get()
+}
+
+// isHTML reports whether a document content type is one kage renders and saves as
+// a page. HTML and XHTML qualify; an empty type is treated as HTML so an unlabelled
+// response still renders. Anything else (a zip, a CSV, a PDF, a bare image or
+// JSON) is an asset that reached the page worker because its link carried no file
+// extension to classify it by.
+func isHTML(contentType string) bool {
+	mt := strings.ToLower(strings.TrimSpace(contentType))
+	if i := strings.IndexByte(mt, ';'); i >= 0 {
+		mt = strings.TrimSpace(mt[:i])
+	}
+	return mt == "" || mt == "text/html" || mt == "application/xhtml+xml"
+}
+
 // settle waits for the network to go quiet for d, recovering from any rod
 // panic and capping the wait so a chatty page can never hang the worker.
 func settle(page *rod.Page, d time.Duration) {

browser/pool_test.go

@@ -2,8 +2,10 @@ package browser
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -17,6 +19,70 @@ func TestLookChromeReadsEnv(t *testing.T) {
 	}
 }
 
+func TestEnvBool(t *testing.T) {
+	cases := []struct {
+		in      string
+		set     bool
+		wantVal bool
+		wantOk  bool
+	}{
+		{"", false, false, false},
+		{"1", true, true, true},
+		{"true", true, true, true},
+		{"TRUE", true, true, true},
+		{"yes", true, true, true},
+		{"on", true, true, true},
+		{"0", true, false, true},
+		{"false", true, false, true},
+		{"off", true, false, true},
+		{"no", true, false, true},
+		{"docker", true, true, true},   // any other non-empty value is true
+		{"  true  ", true, true, true}, // trimmed
+	}
+	for _, c := range cases {
+		if c.set {
+			t.Setenv("KAGE_TEST_BOOL", c.in)
+		} else {
+			_ = os.Unsetenv("KAGE_TEST_BOOL")
+		}
+		val, ok := envBool("KAGE_TEST_BOOL")
+		if val != c.wantVal || ok != c.wantOk {
+			t.Errorf("envBool(%q) = (%v, %v); want (%v, %v)", c.in, val, ok, c.wantVal, c.wantOk)
+		}
+	}
+}
+
+func TestDisableSandboxDefaultKeepsItOn(t *testing.T) {
+	// Not in a container and not root, the sandbox stays on. (When the test
+	// itself runs as root, e.g. some CI containers, "root" is the honest
+	// reason; accept that rather than asserting a false negative.)
+	t.Setenv("IN_DOCKER", "")
+	off, reason := disableSandbox()
+	if isRoot() || inContainer() {
+		if !off {
+			t.Errorf("disableSandbox() = false as root/container; want true")
+		}
+		return
+	}
+	if off {
+		t.Errorf("disableSandbox() = true (%q) on a normal host; want sandbox kept on", reason)
+	}
+}
+
+func TestInContainerHonorsEnv(t *testing.T) {
+	t.Setenv("IN_DOCKER", "1")
+	if !inContainer() {
+		t.Errorf("inContainer() = false with IN_DOCKER=1; want true")
+	}
+}
+
+func TestDisableSandboxContainer(t *testing.T) {
+	t.Setenv("IN_DOCKER", "true")
+	if off, reason := disableSandbox(); !off || reason != "container" {
+		t.Errorf("in container: got (%v, %q); want (true, container)", off, reason)
+	}
+}
+
 func TestRenderCapturesFinalDOM(t *testing.T) {
 	if testing.Short() {
 		t.Skip("render test drives Chrome; skipped under -short")
@@ -50,3 +116,86 @@ func TestRenderCapturesFinalDOM(t *testing.T) {
 		t.Errorf("render did not capture the JS-built DOM:\n%s", res.HTML)
 	}
 }
+
+func TestIsHTML(t *testing.T) {
+	cases := []struct {
+		ct   string
+		want bool
+	}{
+		{"text/html", true},
+		{"text/html; charset=utf-8", true},
+		{"TEXT/HTML", true},
+		{"  text/html ", true},
+		{"application/xhtml+xml", true},
+		{"", true}, // unknown: render rather than misclassify
+		{"application/zip", false},
+		{"text/csv", false},
+		{"application/pdf", false},
+		{"image/png", false},
+		{"application/json", false},
+		{"application/octet-stream", false},
+	}
+	for _, c := range cases {
+		if got := isHTML(c.ct); got != c.want {
+			t.Errorf("isHTML(%q) = %v, want %v", c.ct, got, c.want)
+		}
+	}
+}
+
+func TestRenderRoutesNonHTML(t *testing.T) {
+	if testing.Short() {
+		t.Skip("render test drives Chrome; skipped under -short")
+	}
+	if _, ok := LookChrome(); !ok {
+		t.Skip("no Chrome/Chromium found; skipping render test")
+	}
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/page":
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			_, _ = w.Write([]byte(`<!doctype html><html><body><p>a real page</p></body></html>`))
+		case "/file.zip", "/download":
+			// A binary served with no useful extension on the path, the shape that
+			// makes Chrome download to ~/Downloads when navigated to (issue #32).
+			w.Header().Set("Content-Type", "application/zip")
+			_, _ = w.Write([]byte("PK\x03\x04 not really a zip"))
+		case "/data":
+			w.Header().Set("Content-Type", "text/csv")
+			_, _ = w.Write([]byte("a,b\n1,2\n"))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	p := New(Options{Headless: true, Workers: 1, Settle: 300 * time.Millisecond, RenderTimeout: 20 * time.Second})
+	defer func() { _ = p.Close() }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// A real HTML page renders as before.
+	if res, err := p.Render(ctx, srv.URL+"/page"); err != nil {
+		t.Errorf("render HTML page: %v", err)
+	} else if !strings.Contains(res.HTML, "a real page") {
+		t.Errorf("HTML page did not render:\n%s", res.HTML)
+	}
+
+	// Non-HTML navigation targets come back as *ErrNotHTML so the caller can route
+	// them to the asset downloader instead of saving a broken page or downloading.
+	for _, tc := range []struct{ path, wantCT string }{
+		{"/download", "application/zip"},
+		{"/data", "text/csv"},
+	} {
+		_, err := p.Render(ctx, srv.URL+tc.path)
+		var notHTML *ErrNotHTML
+		if !errors.As(err, &notHTML) {
+			t.Errorf("Render(%s) error = %v, want *ErrNotHTML", tc.path, err)
+			continue
+		}
+		if !strings.Contains(notHTML.ContentType, tc.wantCT) {
+			t.Errorf("Render(%s) content type = %q, want %q", tc.path, notHTML.ContentType, tc.wantCT)
+		}
+	}
+}

cli/clone.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -15,33 +16,36 @@ import (
 
 // cloneFlags holds the parsed flag values for one invocation.
 type cloneFlags struct {
-	out          string
-	reserved     string
-	workers      int
-	assetWorkers int
-	browserPages int
-	maxPages     int
-	maxDepth     int
-	traversal    string
-	maxAssetMB   int64
-	timeout      time.Duration
-	settle       time.Duration
-	renderTO     time.Duration
-	scroll       bool
-	userAgent    string
-	subdomains   bool
-	scopePrefix  string
-	exclude      []string
-	noRobots     bool
-	noSitemap    bool
-	headful      bool
-	keepNoscript bool
-	chromeBin    string
-	controlURL   string
-	noResume     bool
-	refresh      bool
-	force        bool
-	quiet        bool
+	out           string
+	reserved      string
+	workers       int
+	assetWorkers  int
+	browserPages  int
+	maxPages      int
+	maxDepth      int
+	traversal     string
+	maxAssetMB    int64
+	keepMedia     bool
+	skipExt       []string
+	allAssetHosts bool
+	timeout       time.Duration
+	settle        time.Duration
+	renderTO      time.Duration
+	scroll        bool
+	userAgent     string
+	subdomains    bool
+	scopePrefix   string
+	exclude       []string
+	noRobots      bool
+	noSitemap     bool
+	headful       bool
+	keepNoscript  bool
+	chromeBin     string
+	controlURL    string
+	noResume      bool
+	refresh       bool
+	force         bool
+	quiet         bool
 }
 
 func newCloneCmd() *cobra.Command {
@@ -66,7 +70,10 @@ func newCloneCmd() *cobra.Command {
 	fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "stop after N pages (0 = unlimited)")
 	fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)")
 	fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order: bfs or dfs")
-	fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB")
+	fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB (left on the live web)")
+	fs.BoolVar(&f.keepMedia, "keep-media", false, "download bulk media, installers, and PDFs instead of leaving them remote")
+	fs.StringSliceVar(&f.skipExt, "skip-ext", nil, "extra asset extensions to leave remote, e.g. .svg (repeatable)")
+	fs.BoolVar(&f.allAssetHosts, "all-asset-hosts", false, "localize assets from any host, not just the seed's domain")
 	fs.DurationVar(&f.timeout, "timeout", 30*time.Second, "per-request timeout")
 	fs.DurationVar(&f.settle, "settle", 1500*time.Millisecond, "network-idle quiet period before snapshot")
 	fs.DurationVar(&f.renderTO, "render-timeout", 30*time.Second, "hard cap per page render")
@@ -104,6 +111,23 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
 	cfg.MaxDepth = f.maxDepth
 	cfg.Traversal = f.traversal
 	cfg.MaxAssetBytes = f.maxAssetMB << 20
+	cfg.AssetSameDomain = !f.allAssetHosts
+	if f.keepMedia {
+		cfg.SkipAssetExts = map[string]bool{}
+	}
+	for _, e := range f.skipExt {
+		e = strings.ToLower(strings.TrimSpace(e))
+		if e == "" {
+			continue
+		}
+		if !strings.HasPrefix(e, ".") {
+			e = "." + e
+		}
+		if cfg.SkipAssetExts == nil {
+			cfg.SkipAssetExts = map[string]bool{}
+		}
+		cfg.SkipAssetExts[e] = true
+	}
 	cfg.Timeout = f.timeout
 	cfg.Settle = f.settle
 	cfg.RenderTimeout = f.renderTO
@@ -172,23 +196,56 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
 	return nil
 }
 
-// progressLine renders the single-line live counter.
+// progressLine renders the single-line live counter. "pages" is the count of
+// real pages (distinct paths); when a faceted site spawns query-string variants
+// they are shown separately so the page number stays easy to read.
 func progressLine(p clone.Progress) string {
+	if variants := p.Pages - p.PagePaths; variants > 0 {
+		return styleDim.Render(fmt.Sprintf("pages %d  variants %d  assets %d  errors %d  skipped %d",
+			p.PagePaths, variants, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
+	}
 	return styleDim.Render(fmt.Sprintf("pages %d  assets %d  errors %d  skipped %d",
-		p.Pages, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
+		p.PagePaths, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
 }
 
 // printSummary prints the final tally and where the mirror landed.
 func printSummary(res clone.Result) {
 	fmt.Fprintln(os.Stderr, styleOK.Render("done")+" "+styleTitle.Render(res.OutDir))
 	fmt.Fprintf(os.Stderr, "  %s %d   %s %d\n",
-		styleAccent.Render("pages"), res.Pages,
+		styleAccent.Render("pages"), res.PagePaths,
 		styleAccent.Render("assets"), res.Assets)
+	if variants := res.Pages - res.PagePaths; variants > 0 {
+		fmt.Fprintf(os.Stderr, "  %s %d\n", styleDim.Render("query variants"), variants)
+	}
+	if res.PagesLinked > 0 {
+		fmt.Fprintf(os.Stderr, "  %s %d\n", styleDim.Render("deduped (linked)"), res.PagesLinked)
+	}
+	if res.AssetSkipped > 0 {
+		fmt.Fprintf(os.Stderr, "  %s %d\n", styleDim.Render("assets over cap (left remote)"), res.AssetSkipped)
+	}
 	if res.PageErrors+res.AssetErrors > 0 {
 		fmt.Fprintf(os.Stderr, "  %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors)
+		printFailures(res)
 	}
 	if res.Skipped > 0 {
 		fmt.Fprintf(os.Stderr, "  %s %d\n", styleWarn.Render("skipped"), res.Skipped)
 	}
 	fmt.Fprintf(os.Stderr, "  open %s\n", styleAccent.Render("kage serve "+res.OutDir))
 }
+
+// printFailures lists what went wrong, grouped reason and URL, so the error
+// count is actionable instead of opaque. The list is capped during the crawl;
+// when it overflows, say how many more there were.
+func printFailures(res clone.Result) {
+	total := res.PageErrors + res.AssetErrors
+	for _, f := range res.Failures {
+		line := fmt.Sprintf("    %s  %s", styleErr.Render(f.Reason), f.URL)
+		fmt.Fprintln(os.Stderr, line)
+		if f.Referer != "" {
+			fmt.Fprintln(os.Stderr, styleDim.Render("      referenced by "+f.Referer))
+		}
+	}
+	if more := total - int64(len(res.Failures)); more > 0 {
+		fmt.Fprintln(os.Stderr, styleDim.Render(fmt.Sprintf("    ... and %d more", more)))
+	}
+}

cli/open.go

@@ -0,0 +1,66 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tamnd/kage/pack"
+	"github.com/tamnd/kage/viewer"
+	"github.com/tamnd/kage/zim"
+)
+
+func newOpenCmd() *cobra.Command {
+	var addr string
+	var openBrowser bool
+	cmd := &cobra.Command{
+		Use:   "open <file.zim>",
+		Short: "Serve a ZIM archive in your browser for offline reading",
+		Long: "open serves a packed ZIM file over a local HTTP server so you can browse the\n" +
+			"site exactly as it was cloned. It is the read side of kage pack --format zim.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runOpen(cmd.Context(), args[0], addr, openBrowser)
+		},
+	}
+	cmd.Flags().StringVarP(&addr, "addr", "a", "127.0.0.1:8800", "address to listen on")
+	cmd.Flags().BoolVar(&openBrowser, "open", true, "open the default browser")
+	return cmd
+}
+
+func runOpen(ctx context.Context, path, addr string, openBrowser bool) error {
+	r, err := zim.Open(path)
+	if err != nil {
+		return fmt.Errorf("cannot open %q: %w", path, err)
+	}
+	defer func() { _ = r.Close() }()
+
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		return fmt.Errorf("cannot listen on %s: %w", addr, err)
+	}
+	url := "http://" + ln.Addr().String()
+
+	fmt.Fprintln(os.Stderr, styleTitle.Render("kage open")+" "+styleDim.Render(path))
+	fmt.Fprintln(os.Stderr, "  open "+styleAccent.Render(url))
+	if viewer.Native {
+		fmt.Fprintln(os.Stderr, styleDim.Render("  close the window to stop"))
+	} else {
+		fmt.Fprintln(os.Stderr, styleDim.Render("  press Ctrl-C to stop"))
+	}
+
+	srv := &http.Server{Handler: pack.Handler(r)}
+	srvErr := make(chan error, 1)
+	go func() { srvErr <- srv.Serve(ln) }()
+
+	_ = viewer.Show(ctx, viewer.Options{Title: archiveTitle(r), URL: url, Browser: openBrowser})
+	_ = srv.Close()
+	if err := <-srvErr; err != nil && err != http.ErrServerClosed {
+		return err
+	}
+	return nil
+}

cli/pack.go

@@ -0,0 +1,435 @@
+package cli
+
+import (
+	"fmt"
+	"image"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tamnd/kage/clone"
+	"github.com/tamnd/kage/pack"
+)
+
+// packFlags holds the parsed flags for one invocation of kage pack.
+type packFlags struct {
+	format      string
+	out         string
+	base        string
+	app         bool
+	icon        string
+	noCompress  bool
+	title       string
+	description string
+	language    string
+	date        string
+	incremental bool
+}
+
+// cacheSuffix names the cluster-cache sidecar kage writes next to a packed
+// artifact when --incremental is set.
+const cacheSuffix = ".kagecache"
+
+func newPackCmd() *cobra.Command {
+	f := &packFlags{}
+	cmd := &cobra.Command{
+		Use:   "pack <mirror-dir>",
+		Short: "Pack a cloned mirror into a ZIM file or a self-contained viewer",
+		Long: "pack turns a cloned folder into one distributable file. With --format zim\n" +
+			"it writes an open ZIM archive (the format Kiwix uses) that kage open or any\n" +
+			"ZIM reader can browse. With --format binary it appends that archive to a copy\n" +
+			"of kage, producing a single executable that serves the site offline when run.\n" +
+			"Add --app to wrap that executable in a double-click desktop app (a .app bundle\n" +
+			"on macOS, an AppImage-style .AppDir on Linux) with the site's favicon as the icon.\n" +
+			"Add --incremental to keep a cache sidecar so re-packing a mirror only compresses\n" +
+			"the clusters that changed, not the whole archive.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runPack(args[0], f)
+		},
+	}
+	fs := cmd.Flags()
+	fs.StringVar(&f.format, "format", "zim", "output format: zim or binary")
+	fs.StringVarP(&f.out, "out", "o", "", "output path (default per format)")
+	fs.StringVar(&f.base, "base", "", "base kage binary for the viewer (default this kage)")
+	fs.BoolVar(&f.app, "app", false, "wrap the viewer in a double-click desktop app (.app on macOS, .AppImage/.AppDir on Linux)")
+	fs.StringVar(&f.icon, "icon", "", "icon file for --app (default the site's favicon)")
+	fs.BoolVar(&f.noCompress, "no-compress", false, "store every cluster raw, no zstd")
+	fs.BoolVar(&f.incremental, "incremental", false, "reuse compression from a cache sidecar so re-packing a mirror only compresses what changed")
+	fs.StringVar(&f.title, "title", "", "archive title (default the main page's <title>)")
+	fs.StringVar(&f.description, "description", "", "archive description")
+	fs.StringVar(&f.language, "language", "eng", "archive language code")
+	fs.StringVar(&f.date, "date", time.Now().UTC().Format("2006-01-02"), "archive date (YYYY-MM-DD)")
+	return cmd
+}
+
+func runPack(mirrorArg string, f *packFlags) error {
+	dir := resolveMirror(mirrorArg)
+	zopts := pack.ZIMOptions{
+		Out:         f.out,
+		NoCompress:  f.noCompress,
+		Title:       f.title,
+		Description: f.description,
+		Language:    f.language,
+		Date:        f.date,
+		Version:     Version,
+	}
+
+	// --app wraps the packed viewer in a desktop bundle. It builds on the binary
+	// format, so it owns the flow rather than being one more --format value.
+	if f.app {
+		return runPackApp(dir, f, zopts)
+	}
+
+	switch f.format {
+	case "zim":
+		out := f.out
+		if out == "" {
+			out = filepath.Base(dir) + ".zim"
+		}
+		zopts.Out = out
+		var st pack.PackStats
+		if f.incremental {
+			zopts.CachePath = out + cacheSuffix
+			zopts.Stats = &st
+		}
+		outPath, size, err := pack.BuildZIM(dir, zopts)
+		if err != nil {
+			return err
+		}
+		printPackResult(outPath, size)
+		printCacheLine(f.incremental, st)
+		fmt.Fprintf(os.Stderr, "  open %s\n", styleAccent.Render("kage open "+outPath))
+		return nil
+
+	case "binary":
+		target := resolveTargetOS(f.base)
+		out := f.out
+		if out == "" {
+			out = defaultBinaryName(dir)
+		}
+		// A Windows viewer must end in .exe to run, whether the name came from
+		// --out or the default, so make sure it does.
+		if target == "windows" && !strings.HasSuffix(strings.ToLower(out), ".exe") {
+			out += ".exe"
+		}
+		var st pack.PackStats
+		if f.incremental {
+			zopts.CachePath = out + cacheSuffix
+			zopts.Stats = &st
+		}
+		zbytes, err := pack.BuildZIMBytes(dir, zopts)
+		if err != nil {
+			return err
+		}
+		path, size, err := pack.BuildBinary(zbytes, pack.BinaryOptions{Out: out, Base: f.base})
+		if err != nil {
+			return err
+		}
+		printPackResult(path, size)
+		printCacheLine(f.incremental, st)
+		printRunHint(path, target)
+		return nil
+
+	default:
+		return fmt.Errorf("unknown --format %q (want zim or binary)", f.format)
+	}
+}
+
+// runPackApp builds a double-clickable desktop app around the packed viewer,
+// shaped for whichever OS the base targets: a .app bundle on macOS, an
+// AppImage-style .AppDir on Linux. Windows needs no bundle (the .exe is the
+// app), so it is redirected to --format binary with a GUI base.
+func runPackApp(dir string, f *packFlags, zopts pack.ZIMOptions) error {
+	target := resolveTargetOS(f.base)
+	switch target {
+	case "windows":
+		return fmt.Errorf("a Windows app is just the .exe, with no bundle to build: use --format binary and a GUI base (kage built with -ldflags -H=windowsgui)")
+	case "":
+		if f.base != "" {
+			return fmt.Errorf("--app could not tell which OS %q is for; pass a macOS or Linux kage as --base", f.base)
+		}
+		// No base and an unknown runtime: fall through with the host's GOOS.
+		target = runtime.GOOS
+	}
+
+	prog := defaultBinaryName(dir)
+	name := f.title
+	if name == "" {
+		name = prog
+	}
+	icon, iconSrc, err := resolveIcon(dir, f.icon)
+	if err != nil {
+		return err
+	}
+	var st pack.PackStats
+	if f.incremental {
+		zopts.CachePath = prog + cacheSuffix
+		zopts.Stats = &st
+	}
+	zbytes, err := pack.BuildZIMBytes(dir, zopts)
+	if err != nil {
+		return err
+	}
+	printCacheLine(f.incremental, st)
+
+	switch target {
+	case "darwin":
+		return packMacApp(zbytes, dir, f, prog, name, icon, iconSrc)
+	case "linux":
+		return packLinuxApp(zbytes, dir, f, prog, name, icon, iconSrc)
+	default:
+		return fmt.Errorf("--app supports macOS and Linux bases; %s is not one of them", osLabel(target))
+	}
+}
+
+// packMacApp writes the .app bundle and prints how to launch it.
+func packMacApp(zbytes []byte, dir string, f *packFlags, prog, name string, icon image.Image, iconSrc string) error {
+	out := f.out
+	if out == "" {
+		out = prog + ".app"
+	} else if !strings.HasSuffix(strings.ToLower(out), ".app") {
+		out += ".app"
+	}
+	path, size, err := pack.BuildApp(zbytes, pack.AppOptions{
+		Out:        out,
+		Base:       f.base,
+		Name:       name,
+		ExecName:   prog,
+		Identifier: bundleID(prog),
+		Version:    appVersion(),
+		Icon:       icon,
+	})
+	if err != nil {
+		return err
+	}
+	printPackResult(path, size)
+	printIconLine(iconSrc)
+	fmt.Fprintf(os.Stderr, "  double-click %s to open the site offline\n", styleAccent.Render(filepath.Base(path)))
+	if f.base == "" {
+		fmt.Fprintln(os.Stderr, styleDim.Render("  (built around this kage; pass --base a webview build to open a native window instead of the browser)"))
+	}
+	fmt.Fprintln(os.Stderr, styleDim.Render("  (macOS may quarantine it: xattr -dr com.apple.quarantine "+path+")"))
+	return nil
+}
+
+// packLinuxApp writes the .AppDir and, when appimagetool is installed, folds it
+// into a single double-clickable .AppImage.
+func packLinuxApp(zbytes []byte, dir string, f *packFlags, prog, name string, icon image.Image, iconSrc string) error {
+	out := f.out
+	if out == "" {
+		out = prog + ".AppDir"
+	} else if !strings.HasSuffix(out, ".AppDir") {
+		out += ".AppDir"
+	}
+	path, size, hasIcon, err := pack.BuildAppDir(zbytes, pack.LinuxAppOptions{
+		Out:      out,
+		Base:     f.base,
+		Name:     name,
+		ExecName: prog,
+		Comment:  f.description,
+		Version:  appVersion(),
+		Icon:     icon,
+	})
+	if err != nil {
+		return err
+	}
+	printPackResult(path, size)
+	printIconLine(iconSrc)
+
+	// appimagetool turns the directory into one portable file. It needs an icon,
+	// so only attempt it when the mirror gave us one.
+	if hasIcon {
+		if img, ok := tryAppImage(path, prog); ok {
+			fmt.Fprintf(os.Stderr, "  built %s\n", styleTitle.Render(img))
+			fmt.Fprintf(os.Stderr, "  double-click %s to open the site offline\n", styleAccent.Render(filepath.Base(img)))
+			return nil
+		}
+	}
+	fmt.Fprintf(os.Stderr, "  run %s to open the site offline\n", styleAccent.Render("./"+filepath.Join(filepath.Base(path), "AppRun")))
+	fmt.Fprintln(os.Stderr, styleDim.Render("  (install appimagetool to fold this .AppDir into one double-clickable .AppImage)"))
+	return nil
+}
+
+// tryAppImage runs appimagetool over the AppDir if it is installed, returning
+// the .AppImage path on success. A missing tool or a build failure is not fatal:
+// the caller falls back to the AppDir.
+func tryAppImage(appDir, prog string) (string, bool) {
+	tool, err := exec.LookPath("appimagetool")
+	if err != nil {
+		return "", false
+	}
+	out := prog + ".AppImage"
+	cmd := exec.Command(tool, appDir, out)
+	// appimagetool reads the target arch from the AppRun ELF; suppress its noisy
+	// progress so kage's own output stays clean, but surface a real failure.
+	if err := cmd.Run(); err != nil {
+		return "", false
+	}
+	if _, err := os.Stat(out); err != nil {
+		return "", false
+	}
+	return out, true
+}
+
+func printIconLine(iconSrc string) {
+	if iconSrc != "" {
+		fmt.Fprintf(os.Stderr, "  icon %s\n", styleDim.Render(iconSrc))
+	}
+}
+
+// resolveIcon picks the bundle icon: an explicit --icon path if given (an error
+// there is fatal, since the user asked for that file), otherwise the site's
+// favicon discovered in the mirror. A mirror with no usable icon is fine; the
+// bundle just ships without a custom one.
+func resolveIcon(dir, iconFlag string) (img image.Image, src string, err error) {
+	if iconFlag != "" {
+		img, err = pack.DecodeIcon(iconFlag)
+		if err != nil {
+			return nil, "", err
+		}
+		return img, iconFlag, nil
+	}
+	if img, src, ok := pack.FindIcon(dir); ok {
+		return img, src, nil
+	}
+	return nil, "", nil
+}
+
+// bundleID builds a reverse-DNS CFBundleIdentifier from the program name,
+// keeping only characters Apple allows in an identifier.
+func bundleID(prog string) string {
+	var b strings.Builder
+	for _, r := range strings.ToLower(prog) {
+		switch {
+		case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-':
+			b.WriteRune(r)
+		default:
+			b.WriteRune('-')
+		}
+	}
+	id := strings.Trim(b.String(), "-")
+	if id == "" {
+		id = "app"
+	}
+	return "com.kage." + id
+}
+
+// appVersion uses kage's own version for the bundle, falling back to 1.0 for a
+// dev build whose version is the default "dev".
+func appVersion() string {
+	if Version == "" || Version == "dev" {
+		return "1.0"
+	}
+	return strings.TrimPrefix(Version, "v")
+}
+
+// resolveMirror accepts either a path to a mirror dir or a bare host. A bare
+// host that is not a directory in the working dir is resolved against the
+// default out dir, so "kage pack paulgraham.com" works right after a clone.
+func resolveMirror(arg string) string {
+	if info, err := os.Stat(arg); err == nil && info.IsDir() {
+		return arg
+	}
+	candidate := filepath.Join(clone.DefaultOutDir(), arg)
+	if info, err := os.Stat(candidate); err == nil && info.IsDir() {
+		return candidate
+	}
+	return arg
+}
+
+// defaultBinaryName derives a clean program name from the mirror's host by
+// stripping a trailing dot-suffix (paulgraham.com -> paulgraham). The caller
+// appends .exe for Windows targets.
+func defaultBinaryName(dir string) string {
+	host := filepath.Base(dir)
+	if i := strings.IndexByte(host, '.'); i > 0 {
+		return host[:i]
+	}
+	return host
+}
+
+// resolveTargetOS reports which OS the packed viewer will run on. With no
+// --base it is this kage's OS; with one, we sniff the base's executable header
+// so detection does not hinge on the file being named ".exe". If the header is
+// unrecognised we fall back to that name heuristic.
+func resolveTargetOS(base string) string {
+	if base == "" {
+		return runtime.GOOS
+	}
+	if os := pack.SniffOS(base); os != "" {
+		return os
+	}
+	if strings.HasSuffix(strings.ToLower(base), ".exe") {
+		return "windows"
+	}
+	return ""
+}
+
+func printPackResult(path string, size int64) {
+	fmt.Fprintln(os.Stderr, styleOK.Render("packed")+" "+styleTitle.Render(path))
+	fmt.Fprintf(os.Stderr, "  %s %s\n", styleAccent.Render("size"), humanBytes(size))
+}
+
+// printCacheLine reports how much compression the incremental cache reused. On
+// the first pack everything is compressed fresh; on a re-pack after a small
+// change most clusters are reused and only the rest are compressed.
+func printCacheLine(incremental bool, st pack.PackStats) {
+	if !incremental {
+		return
+	}
+	total := st.ClustersReused + st.ClustersCompressed
+	fmt.Fprintf(os.Stderr, "  %s %d reused, %d compressed of %d clusters\n",
+		styleDim.Render("cache"), st.ClustersReused, st.ClustersCompressed, total)
+}
+
+func printRunHint(path, target string) {
+	rel := path
+	if !strings.ContainsAny(path, "/\\") {
+		rel = "./" + path
+	}
+	// A viewer built for another OS cannot run here, so say where it goes
+	// instead of printing a run command that would not work.
+	if target != "" && target != runtime.GOOS {
+		fmt.Fprintf(os.Stderr, "  this is a %s viewer; copy %s to that machine to run it\n",
+			osLabel(target), styleAccent.Render(filepath.Base(path)))
+		return
+	}
+	fmt.Fprintf(os.Stderr, "  run %s to view the site offline\n", styleAccent.Render(rel))
+	if target == "darwin" {
+		fmt.Fprintln(os.Stderr, styleDim.Render("  (macOS may quarantine it: xattr -d com.apple.quarantine "+rel+")"))
+	}
+}
+
+// osLabel turns a GOOS value into a friendly name for the run hint.
+func osLabel(goos string) string {
+	switch goos {
+	case "windows":
+		return "Windows"
+	case "darwin":
+		return "macOS"
+	case "linux":
+		return "Linux"
+	default:
+		return goos
+	}
+}
+
+// humanBytes renders a byte count in B, KiB, MiB, or GiB.
+func humanBytes(n int64) string {
+	const unit = 1024
+	if n < unit {
+		return fmt.Sprintf("%d B", n)
+	}
+	div, exp := int64(unit), 0
+	for x := n / unit; x >= unit; x /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
+}

cli/parquet.go

@@ -0,0 +1,94 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tamnd/kage/dataset"
+)
+
+// newParquetCmd groups the two columnar conversions: a ZIM archive out to a
+// Parquet table, and a Parquet table back to a ZIM archive. The table is a flat
+// one-row-per-entry shape ready to publish as a dataset, and the round trip is
+// lossless.
+func newParquetCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "parquet",
+		Short: "Convert a ZIM archive to a Parquet dataset and back",
+		Long: "parquet converts a packed ZIM archive into a columnar Parquet table, one row\n" +
+			"per entry with clear columns (url, mime, title, content, extracted text), and\n" +
+			"converts such a table back into a ZIM. The table is the shape a dataset host\n" +
+			"like Hugging Face expects, and the conversion is lossless: a ZIM round-tripped\n" +
+			"through Parquet reproduces every entry, its metadata, and the main page.",
+	}
+	cmd.AddCommand(newParquetExportCmd())
+	cmd.AddCommand(newParquetImportCmd())
+	return cmd
+}
+
+func newParquetExportCmd() *cobra.Command {
+	var out string
+	cmd := &cobra.Command{
+		Use:   "export <file.zim>",
+		Short: "Write a Parquet table from a ZIM archive",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			in := args[0]
+			if out == "" {
+				out = strings.TrimSuffix(in, filepath.Ext(in)) + ".parquet"
+			}
+			st, err := dataset.ZIMToParquet(in, out, Version)
+			if err != nil {
+				return err
+			}
+			printDatasetResult("exported", out)
+			printDatasetStats(st, out)
+			return nil
+		},
+	}
+	cmd.Flags().StringVarP(&out, "out", "o", "", "output path (default <name>.parquet)")
+	return cmd
+}
+
+func newParquetImportCmd() *cobra.Command {
+	var out string
+	cmd := &cobra.Command{
+		Use:   "import <file.parquet>",
+		Short: "Rebuild a ZIM archive from a Parquet table",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			in := args[0]
+			if out == "" {
+				out = strings.TrimSuffix(in, filepath.Ext(in)) + ".zim"
+			}
+			st, err := dataset.ParquetToZIM(in, out, Version)
+			if err != nil {
+				return err
+			}
+			printDatasetResult("imported", out)
+			printDatasetStats(st, out)
+			fmt.Fprintf(os.Stderr, "  open %s\n", styleAccent.Render("kage open "+out))
+			return nil
+		},
+	}
+	cmd.Flags().StringVarP(&out, "out", "o", "", "output path (default <name>.zim)")
+	return cmd
+}
+
+func printDatasetResult(verb, path string) {
+	fmt.Fprintln(os.Stderr, styleOK.Render(verb)+" "+styleTitle.Render(path))
+}
+
+func printDatasetStats(st dataset.Stats, path string) {
+	fmt.Fprintf(os.Stderr, "  %s %d   %s %d\n",
+		styleAccent.Render("rows"), st.Rows,
+		styleAccent.Render("redirects"), st.Redirects)
+	fmt.Fprintf(os.Stderr, "  %s %s content\n", styleDim.Render("content"), humanBytes(st.ContentBytes))
+	if fi, err := os.Stat(path); err == nil {
+		fmt.Fprintf(os.Stderr, "  %s %s\n", styleAccent.Render("size"), humanBytes(fi.Size()))
+	}
+}

cli/root.go

@@ -7,15 +7,29 @@ package cli
 import (
 	"context"
 	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
 
 	"github.com/charmbracelet/fang"
 	"github.com/spf13/cobra"
+
+	"github.com/tamnd/kage/pack"
+	"github.com/tamnd/kage/viewer"
+	"github.com/tamnd/kage/zim"
 )
 
 // Execute builds the root command and runs it through fang. main passes the
 // signal-aware context so Ctrl-C cancels the in-flight clone and flushes resume
 // state. It returns the process exit code.
 func Execute(ctx context.Context) int {
+	// A kage binary with a ZIM appended runs as an offline viewer for that site,
+	// ignoring its arguments. A normal build has no trailer and falls through.
+	if ra, size, ok := pack.Embedded(); ok {
+		return runEmbeddedViewer(ctx, ra, size)
+	}
+
 	root := newRoot()
 	opts := []fang.Option{
 		fang.WithVersion(Version),
@@ -41,5 +55,55 @@ func newRoot() *cobra.Command {
 	}
 	root.AddCommand(newCloneCmd())
 	root.AddCommand(newServeCmd())
+	root.AddCommand(newPackCmd())
+	root.AddCommand(newOpenCmd())
+	root.AddCommand(newParquetCmd())
 	return root
 }
+
+// runEmbeddedViewer serves the ZIM appended to this executable on an ephemeral
+// local port and shows it: a native window in the webview build, the system
+// browser otherwise. It runs until the viewer closes or the context is
+// cancelled (Ctrl-C) and ignores all command-line arguments, because a packed
+// binary is the site, not the kage CLI.
+func runEmbeddedViewer(ctx context.Context, ra io.ReaderAt, size int64) int {
+	r, err := zim.NewReader(ra, size)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "kage: corrupt embedded archive:", err)
+		return 1
+	}
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "kage: cannot start viewer:", err)
+		return 1
+	}
+	url := "http://" + ln.Addr().String()
+	if viewer.Native {
+		fmt.Fprintln(os.Stderr, "opening offline site (close the window to stop)")
+	} else {
+		fmt.Fprintln(os.Stderr, "serving offline site at "+url+"  (Ctrl-C to stop)")
+	}
+
+	srv := &http.Server{Handler: pack.Handler(r)}
+	srvErr := make(chan error, 1)
+	go func() { srvErr <- srv.Serve(ln) }()
+
+	// Show blocks until the window closes (native) or ctx is cancelled (browser);
+	// either way, tear the server down afterwards.
+	_ = viewer.Show(ctx, viewer.Options{Title: archiveTitle(r), URL: url, Browser: true})
+	_ = srv.Close()
+	if err := <-srvErr; err != nil && err != http.ErrServerClosed {
+		fmt.Fprintln(os.Stderr, "kage:", err)
+		return 1
+	}
+	return 0
+}
+
+// archiveTitle returns the archive's M/Title metadata for use as a window
+// title, falling back to the empty string (viewer defaults it to "kage").
+func archiveTitle(r *zim.Reader) string {
+	if b, err := r.Get(zim.NamespaceMetadata, "Title"); err == nil {
+		return string(b.Data)
+	}
+	return ""
+}

clone/cloner.go

@@ -2,6 +2,8 @@ package clone
 
 import (
 	"context"
+	"crypto/sha256"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -44,6 +46,9 @@ type Cloner struct {
 	wg         sync.WaitGroup
 	pageJobs   chan pageItem
 	assetJobs  chan assetItem
+
+	muContent   sync.Mutex
+	seenContent map[string]string // sha-256 of page bytes -> first path written
 }
 
 type pageItem struct {
@@ -65,19 +70,20 @@ func New(seed *url.URL, cfg Config, logf Logf) *Cloner {
 	host := seed.Hostname()
 	outRoot := cfg.HostDir(host)
 	return &Cloner{
-		cfg:        cfg,
-		seed:       seed,
-		seedHost:   host,
-		outRoot:    outRoot,
-		statePth:   filepath.Join(outRoot, cfg.Reserved, "state.json"),
-		dl:         asset.NewDownloader(cfg.UserAgent, cfg.Timeout, cfg.MaxAssetBytes),
-		httpC:      &http.Client{Timeout: cfg.Timeout},
-		robots:     robots.AllowAll(),
-		front:      newFrontier(),
-		logf:       logf,
-		seenAssets: map[string]bool{},
-		pageJobs:   make(chan pageItem),
-		assetJobs:  make(chan assetItem),
+		cfg:         cfg,
+		seed:        seed,
+		seedHost:    host,
+		outRoot:     outRoot,
+		statePth:    filepath.Join(outRoot, cfg.Reserved, "state.json"),
+		dl:          asset.NewDownloader(cfg.UserAgent, cfg.Timeout, cfg.MaxAssetBytes),
+		httpC:       &http.Client{Timeout: cfg.Timeout},
+		robots:      robots.AllowAll(),
+		front:       newFrontier(),
+		logf:        logf,
+		seenAssets:  map[string]bool{},
+		seenContent: map[string]string{},
+		pageJobs:    make(chan pageItem),
+		assetJobs:   make(chan assetItem),
 	}
 }
 
@@ -97,6 +103,19 @@ func (c *Cloner) assetKey(u *url.URL) string {
 	return urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
 }
 
+// pagePathKey is the identity of a page ignoring its query string, used to tell
+// a real page apart from its ?q=…/?page=… variants for the progress display.
+// Each variant writes its own file (so the crawl stays complete), but they all
+// fold to one path here.
+func (c *Cloner) pagePathKey(u *url.URL) string {
+	if u.RawQuery == "" {
+		return c.pageKey(u)
+	}
+	cp := *u
+	cp.RawQuery = ""
+	return c.pageKey(&cp)
+}
+
 // Run executes the clone until the frontier drains, MaxPages is hit, or ctx is
 // cancelled (which flushes the resume state). It returns the final Result.
 func (c *Cloner) Run(ctx context.Context) (Result, error) {
@@ -165,7 +184,7 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) {
 		}
 	}
 
-	res := Result{Progress: c.stats.snapshot(), OutDir: c.outRoot}
+	res := Result{Progress: c.stats.snapshot(), OutDir: c.outRoot, Failures: c.stats.recordedFailures()}
 	if ctx.Err() != nil {
 		return res, ctx.Err()
 	}
@@ -236,15 +255,29 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 
 	res, err := c.pool.Render(ctx, j.u.String())
 	if err != nil {
-		c.stats.pageErrors.Add(1)
-		c.logf("page error %s: %v", j.u, err)
+		var notHTML *browser.ErrNotHTML
+		if errors.As(err, &notHTML) {
+			// The URL is not a page but a file (a zip, a CSV, a bare image) that
+			// reached the page worker through an extensionless link. Hand it to the
+			// asset downloader, where the size and media policy decides whether to
+			// localise it or leave it remote, rather than saving a broken page or
+			// letting Chrome download it to the user's Downloads folder (issue #32).
+			c.front.markVisited(key)
+			if c.wantAsset(j.u) {
+				c.enqueueAsset(ctx, j.u, "")
+				c.logf("not a page, fetching as asset (%s): %s", notHTML.ContentType, j.u.String())
+			} else {
+				c.logf("not a page, left on the live web (%s): %s", notHTML.ContentType, j.u.String())
+			}
+			return
+		}
+		c.failPage(j.u.String(), fmt.Errorf("render: %w", err))
 		return
 	}
 
 	root, err := html.Parse(strings.NewReader(res.HTML))
 	if err != nil {
-		c.stats.pageErrors.Add(1)
-		c.logf("parse error %s: %v", j.u, err)
+		c.failPage(j.u.String(), fmt.Errorf("parse: %w", err))
 		return
 	}
 
@@ -261,6 +294,9 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 			}
 			return u.String() // external page link stays on the live web
 		default: // Asset
+			if !c.wantAsset(u) {
+				return u.String() // off-domain or bulk media: leave it on the live web
+			}
 			c.enqueueAsset(ctx, u, j.u.String())
 			local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
 			return urlx.Rel(fileDir, local)
@@ -275,16 +311,16 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 
 	var buf strings.Builder
 	if err := html.Render(&buf, root); err != nil {
-		c.stats.pageErrors.Add(1)
+		c.failPage(j.u.String(), fmt.Errorf("render html: %w", err))
 		return
 	}
-	if err := c.writeFile(localFile, []byte(buf.String())); err != nil {
-		c.stats.pageErrors.Add(1)
-		c.logf("write error %s: %v", localFile, err)
+	deduped, err := c.writePage(localFile, []byte(buf.String()))
+	if err != nil {
+		c.failPage(j.u.String(), fmt.Errorf("write %s: %w", localFile, err))
 		return
 	}
 	c.front.markVisited(key)
-	c.stats.pages.Add(1)
+	c.stats.recordPage(c.pagePathKey(j.u), deduped)
 }
 
 // processAsset downloads one asset, rewriting CSS references on the way, and
@@ -295,8 +331,14 @@ func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
 	}
 	res, err := c.dl.Get(ctx, j.u, j.referer)
 	if err != nil {
-		c.stats.assetErrors.Add(1)
-		c.logf("asset error %s: %v", j.u, err)
+		if errors.Is(err, asset.ErrTooLarge) {
+			// Over the size cap: leave it out rather than save a truncated
+			// fragment. Count it as skipped, not failed, so the run is clean.
+			c.stats.assetSkipped.Add(1)
+			c.logf("asset skipped (over %d MB): %s", c.cfg.MaxAssetBytes>>20, j.u.String())
+			return
+		}
+		c.failAsset(j.u.String(), j.referer, err)
 		return
 	}
 
@@ -305,6 +347,9 @@ func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
 	if res.IsCSS {
 		fileDir := urlx.Dir(localFile)
 		cssSink := func(u *url.URL, _ urlx.Kind) string {
+			if !c.wantAsset(u) {
+				return u.String() // off-domain or bulk media: leave it on the live web
+			}
 			c.enqueueAsset(ctx, u, j.u.String())
 			local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
 			return urlx.Rel(fileDir, local)
@@ -312,13 +357,54 @@ func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
 		body = asset.RewriteCSS(body, j.u, cssSink)
 	}
 	if err := c.writeFile(localFile, body); err != nil {
-		c.stats.assetErrors.Add(1)
-		c.logf("write error %s: %v", localFile, err)
+		c.failAsset(j.u.String(), j.referer, fmt.Errorf("write %s: %w", localFile, err))
 		return
 	}
 	c.stats.assets.Add(1)
 }
 
+// failAsset records and logs a failed asset, naming the page that referenced it
+// so a 403 or 404 is traceable back to where it came from. The reason is
+// classified (HTTP status, timeout, or other) for a readable line.
+func (c *Cloner) failAsset(u, referer string, err error) {
+	c.stats.assetErrors.Add(1)
+	reason := classifyError(err)
+	c.stats.recordFailure(Failure{Kind: "asset", URL: u, Referer: referer, Reason: reason})
+	if referer != "" {
+		c.logf("asset error: %s\n    %s\n    referenced by %s", reason, u, referer)
+	} else {
+		c.logf("asset error: %s\n    %s", reason, u)
+	}
+}
+
+// failPage records and logs a failed page.
+func (c *Cloner) failPage(u string, err error) {
+	c.stats.pageErrors.Add(1)
+	reason := classifyError(err)
+	c.stats.recordFailure(Failure{Kind: "page", URL: u, Reason: reason})
+	c.logf("page error: %s\n    %s", reason, u)
+}
+
+// classifyError turns an error into a short, human-readable reason for the log
+// and the final report: an HTTP status with its name, a timeout, a cancellation,
+// or the underlying message otherwise.
+func classifyError(err error) string {
+	if err == nil {
+		return ""
+	}
+	var se *asset.StatusError
+	if errors.As(err, &se) {
+		return se.Error()
+	}
+	switch {
+	case errors.Is(err, context.DeadlineExceeded):
+		return "timed out"
+	case errors.Is(err, context.Canceled):
+		return "cancelled"
+	}
+	return err.Error()
+}
+
 // enqueuePage offers a page URL to the frontier, honouring the visited set, the
 // depth cap, and the page budget. It reports whether the page was newly queued.
 func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool {
@@ -351,6 +437,22 @@ func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool {
 	return true
 }
 
+// wantAsset reports whether an asset should be downloaded and localized, or left
+// pointing at its live URL. kage skips two classes by default: assets on a host
+// outside the seed's registrable domain (a third-party tracker, an unrelated
+// CDN), and bulk media, installers, and archives whose extension is in the skip
+// set. Both are decisions a page worker can make from the URL alone, before any
+// download, so the rewritten HTML simply keeps the remote link.
+func (c *Cloner) wantAsset(u *url.URL) bool {
+	if c.cfg.AssetSameDomain && !urlx.SameRegistrableDomain(c.seed, u) {
+		return false
+	}
+	if c.cfg.SkipAssetExts[urlx.Ext(u)] {
+		return false
+	}
+	return true
+}
+
 // enqueueAsset offers an asset URL for download, deduping by canonical URL.
 func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) {
 	key := c.assetKey(u)
@@ -372,6 +474,49 @@ func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) {
 	}()
 }
 
+// writePage writes a rendered page, deduping by content. The first page with a
+// given byte content is written normally; a later page with identical bytes is
+// stored as a hard link to that first file, so the same content never occupies
+// disk twice (a faceted site whose ?q=… variants all render the same page is the
+// motivating case). It reports whether this write was deduped. If hard links are
+// unsupported, it falls back to writing the bytes, so correctness never depends
+// on the link succeeding.
+func (c *Cloner) writePage(relSlash string, data []byte) (bool, error) {
+	sum := sha256.Sum256(data)
+	h := string(sum[:])
+
+	c.muContent.Lock()
+	canon, seen := c.seenContent[h]
+	if !seen {
+		c.seenContent[h] = relSlash
+	}
+	c.muContent.Unlock()
+
+	if seen && canon != relSlash {
+		if err := c.linkFile(canon, relSlash); err == nil {
+			return true, nil
+		}
+		// The canonical file may not be on disk yet (a concurrent first write)
+		// or the filesystem may not support links; write the bytes instead.
+	}
+	return false, c.writeFile(relSlash, data)
+}
+
+// linkFile hard-links the already-written canonical file to target, replacing
+// any file already at target. Both paths are confined to the mirror root.
+func (c *Cloner) linkFile(canonSlash, targetSlash string) error {
+	canonFull := filepath.Join(c.outRoot, filepath.FromSlash(canonSlash))
+	targetFull := filepath.Join(c.outRoot, filepath.FromSlash(targetSlash))
+	if !strings.HasPrefix(targetFull, filepath.Clean(c.outRoot)+string(os.PathSeparator)) {
+		return fmt.Errorf("refusing to link outside mirror root: %s", targetSlash)
+	}
+	if err := os.MkdirAll(filepath.Dir(targetFull), 0o755); err != nil {
+		return err
+	}
+	_ = os.Remove(targetFull)
+	return os.Link(canonFull, targetFull)
+}
+
 // writeFile writes data to a slash path relative to the mirror root, creating
 // parent directories. The path is cleaned so it can never escape the root.
 func (c *Cloner) writeFile(relSlash string, data []byte) error {

clone/cloner_test.go

@@ -261,6 +261,79 @@ func TestCloneRefreshReRenders(t *testing.T) {
 	}
 }
 
+// TestCloneRoutesNonHTMLToAsset guards issue #32: an extensionless link that
+// turns out to be a file (a zip) is classified as a page up front, but once the
+// page worker sees it is not HTML it must be handed to the asset downloader, not
+// saved as a broken page nor downloaded by Chrome to ~/Downloads.
+func TestCloneRoutesNonHTMLToAsset(t *testing.T) {
+	if testing.Short() {
+		t.Skip("clone test drives Chrome; skipped under -short")
+	}
+	if _, ok := browser.LookChrome(); !ok {
+		t.Skip("no Chrome/Chromium found; skipping clone test")
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		// The link has no extension, so it is queued as a page; the server then
+		// answers it with a zip.
+		_, _ = w.Write([]byte(`<!doctype html><html><body>
+<h1>Home</h1><a href="/download">grab the bundle</a></body></html>`))
+	})
+	mux.HandleFunc("/download", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/zip")
+		_, _ = w.Write([]byte("PK\x03\x04 pretend bundle"))
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	seed, err := urlx.ParseSeed(srv.URL)
+	if err != nil {
+		t.Fatalf("parse seed: %v", err)
+	}
+
+	out := t.TempDir()
+	cfg := DefaultConfig()
+	cfg.OutDir = out
+	cfg.Settle = 300 * time.Millisecond
+	cfg.RenderTimeout = 20 * time.Second
+	cfg.Timeout = 10 * time.Second
+
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+
+	res, err := New(seed, cfg, t.Logf).Run(ctx)
+	if err != nil {
+		t.Fatalf("run: %v", err)
+	}
+
+	root := res.OutDir
+	// The home page is a real page and is written.
+	if !fileExists(filepath.Join(root, "index.html")) {
+		t.Error("home page was not written")
+	}
+	// The zip is NOT saved as a page: no download/index.html exists.
+	if fileExists(filepath.Join(root, "download", "index.html")) {
+		t.Error("non-HTML target was saved as a page")
+	}
+	// The zip is fetched as an asset under the reserved tree instead.
+	if res.Assets < 1 {
+		t.Errorf("expected the zip to be fetched as an asset, assets=%d", res.Assets)
+	}
+	assetDir := filepath.Join(root, cfg.Reserved)
+	if !anyFileUnder(t, assetDir, "download") {
+		t.Error("the zip was not downloaded into the reserved asset tree")
+	}
+	if res.PageErrors != 0 {
+		t.Errorf("a rerouted non-HTML target must not count as a page error, got %d", res.PageErrors)
+	}
+}
+
 func readFile(t *testing.T, path string) string {
 	t.Helper()
 	b, err := os.ReadFile(path)

clone/config.go

@@ -36,6 +36,16 @@ type Config struct {
 	Traversal     string
 	MaxAssetBytes int64
 
+	// AssetSameDomain, when set, localizes only assets whose host shares the
+	// seed's registrable domain (apple.com covers developer.apple.com and
+	// www.apple.com but not cdn-apple.com or an unrelated third party). An
+	// off-domain asset is left pointing at its live URL instead of downloaded.
+	AssetSameDomain bool
+	// SkipAssetExts lists asset extensions (".mp4", ".pdf", ".dmg", …) that are
+	// left on the live web rather than downloaded, so bulk media, installers, and
+	// archives do not bloat the mirror. The reference keeps its remote URL.
+	SkipAssetExts map[string]bool
+
 	Timeout       time.Duration // per HTTP request
 	Settle        time.Duration // network-idle quiet period
 	RenderTimeout time.Duration // hard cap per page render
@@ -70,25 +80,52 @@ type Config struct {
 const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +
 	"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
 
+// DefaultSkipAssetExts returns the asset extensions kage leaves on the live web
+// by default: bulk media, installers, and archives that rarely matter for
+// reading a site offline but dominate its download size (a docs site's WWDC
+// videos, .dmg/.pkg installers, and PDF manuals can be most of the bytes).
+// Page-rendering assets (images, fonts, CSS) are deliberately absent, so the
+// offline pages still look right.
+func DefaultSkipAssetExts() map[string]bool {
+	exts := []string{
+		// Video and audio.
+		".mp4", ".m4v", ".mov", ".avi", ".mkv", ".webm", ".flv", ".wmv",
+		".m3u8", ".ts", ".mp3", ".wav", ".flac", ".aac", ".ogg", ".oga",
+		// Installers and disk images.
+		".dmg", ".pkg", ".exe", ".msi", ".deb", ".rpm", ".appimage", ".iso",
+		// Archives.
+		".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".7z", ".rar",
+		// Documents that download rather than render.
+		".pdf",
+	}
+	m := make(map[string]bool, len(exts))
+	for _, e := range exts {
+		m[e] = true
+	}
+	return m
+}
+
 // DefaultConfig returns the baseline configuration.
 func DefaultConfig() Config {
 	return Config{
-		OutDir:        DefaultOutDir(),
-		Reserved:      urlx.DefaultReserved,
-		Workers:       4,
-		AssetWorkers:  8,
-		BrowserPages:  4,
-		MaxAssetBytes: 25 << 20,
-		Traversal:     "bfs",
-		Timeout:       30 * time.Second,
-		Settle:        1500 * time.Millisecond,
-		RenderTimeout: 30 * time.Second,
-		UserAgent:     DefaultUserAgent,
-		RespectRobots: true,
-		FollowSitemap: true,
-		Headless:      true,
-		Resume:        true,
-		Persist:       true,
+		OutDir:          DefaultOutDir(),
+		Reserved:        urlx.DefaultReserved,
+		Workers:         4,
+		AssetWorkers:    8,
+		BrowserPages:    4,
+		MaxAssetBytes:   25 << 20,
+		AssetSameDomain: true,
+		SkipAssetExts:   DefaultSkipAssetExts(),
+		Traversal:       "bfs",
+		Timeout:         30 * time.Second,
+		Settle:          1500 * time.Millisecond,
+		RenderTimeout:   30 * time.Second,
+		UserAgent:       DefaultUserAgent,
+		RespectRobots:   true,
+		FollowSitemap:   true,
+		Headless:        true,
+		Resume:          true,
+		Persist:         true,
 	}
 }
 

clone/dedup_test.go

@@ -0,0 +1,69 @@
+package clone
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestWritePageDedup checks that identical page bytes are stored once and shared
+// by a hard link, while different bytes are written as their own file.
+func TestWritePageDedup(t *testing.T) {
+	dir := t.TempDir()
+	c := &Cloner{outRoot: dir, seenContent: map[string]string{}}
+
+	body := []byte("<html><body>same page</body></html>")
+
+	if deduped, err := c.writePage("a/index.html", body); err != nil || deduped {
+		t.Fatalf("first write: deduped=%v err=%v, want false/nil", deduped, err)
+	}
+	if deduped, err := c.writePage("b/index.html", body); err != nil || !deduped {
+		t.Fatalf("second identical write: deduped=%v err=%v, want true/nil", deduped, err)
+	}
+	if deduped, err := c.writePage("c/index.html", []byte("<html>other</html>")); err != nil || deduped {
+		t.Fatalf("third different write: deduped=%v err=%v, want false/nil", deduped, err)
+	}
+
+	// The two identical pages must be the same file on disk (one inode).
+	fa, err := os.Stat(filepath.Join(dir, "a/index.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	fb, err := os.Stat(filepath.Join(dir, "b/index.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !os.SameFile(fa, fb) {
+		t.Error("identical pages were not hard-linked to the same file")
+	}
+
+	// The different page must stand alone with its own bytes.
+	got, err := os.ReadFile(filepath.Join(dir, "c/index.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != "<html>other</html>" {
+		t.Errorf("third page content = %q", got)
+	}
+}
+
+// TestRecordPageCounts checks that pages counts every write, pagePaths counts
+// distinct query-stripped paths, and pagesLinked counts deduped writes.
+func TestRecordPageCounts(t *testing.T) {
+	var s stats
+	s.recordPage("showcase/index.html", false)
+	s.recordPage("showcase/index.html", true) // a ?q= variant of the same path
+	s.recordPage("showcase/index.html", true)
+	s.recordPage("about/index.html", false)
+
+	p := s.snapshot()
+	if p.Pages != 4 {
+		t.Errorf("Pages = %d, want 4", p.Pages)
+	}
+	if p.PagePaths != 2 {
+		t.Errorf("PagePaths = %d, want 2", p.PagePaths)
+	}
+	if p.PagesLinked != 2 {
+		t.Errorf("PagesLinked = %d, want 2", p.PagesLinked)
+	}
+}

clone/stats.go

@@ -1,32 +1,104 @@
 package clone
 
-import "sync/atomic"
+import (
+	"sync"
+	"sync/atomic"
+)
+
+// maxRecordedFailures caps how many individual failures Run keeps for the final
+// report, so a huge broken site cannot grow the slice without bound. The error
+// counters still count every failure.
+const maxRecordedFailures = 100
 
 // stats are the live counters of a run, read by the CLI's progress ticker.
 type stats struct {
-	pages       atomic.Int64
-	assets      atomic.Int64
-	pageErrors  atomic.Int64
-	assetErrors atomic.Int64
-	skipped     atomic.Int64 // robots-disallowed or out of budget
+	pages        atomic.Int64 // page documents written (one per output file)
+	pagePaths    atomic.Int64 // distinct URL paths among those, ignoring query
+	pagesLinked  atomic.Int64 // pages stored as a hard link to identical content
+	assets       atomic.Int64
+	assetSkipped atomic.Int64 // assets left on the live web (over the size cap)
+	pageErrors   atomic.Int64
+	assetErrors  atomic.Int64
+	skipped      atomic.Int64 // robots-disallowed or out of budget
+
+	muPaths  sync.Mutex
+	seenPath map[string]struct{}
+
+	muFail   sync.Mutex
+	failures []Failure
 }
 
-// Progress is a snapshot of a run for display.
+// recordPage counts a freshly written page. Every write bumps pages; the first
+// write for a given query-stripped path also bumps pagePaths, so the display can
+// separate real pages from the query-string variants (?q=…, ?page=…) that a
+// single path can spawn by the thousand on a faceted site. deduped marks a page
+// whose bytes were stored as a hard link to identical content already on disk.
+func (s *stats) recordPage(pathKey string, deduped bool) {
+	s.pages.Add(1)
+	if deduped {
+		s.pagesLinked.Add(1)
+	}
+	s.muPaths.Lock()
+	if s.seenPath == nil {
+		s.seenPath = make(map[string]struct{})
+	}
+	if _, ok := s.seenPath[pathKey]; !ok {
+		s.seenPath[pathKey] = struct{}{}
+		s.pagePaths.Add(1)
+	}
+	s.muPaths.Unlock()
+}
+
+// Failure is one thing that went wrong, kept for the end-of-run report so the
+// errors are visible as a list rather than only as a count.
+type Failure struct {
+	Kind    string // "page" or "asset"
+	URL     string
+	Referer string // the page that referenced it, when known
+	Reason  string // e.g. "HTTP 403 Forbidden"
+}
+
+func (s *stats) recordFailure(f Failure) {
+	s.muFail.Lock()
+	if len(s.failures) < maxRecordedFailures {
+		s.failures = append(s.failures, f)
+	}
+	s.muFail.Unlock()
+}
+
+func (s *stats) recordedFailures() []Failure {
+	s.muFail.Lock()
+	defer s.muFail.Unlock()
+	out := make([]Failure, len(s.failures))
+	copy(out, s.failures)
+	return out
+}
+
+// Progress is a snapshot of a run for display. Pages is every page document
+// written (it equals the count of HTML files on disk); PagePaths is how many
+// distinct URL paths those represent once query strings are ignored. The
+// difference, Pages-PagePaths, is the number of query-string variants.
 type Progress struct {
-	Pages       int64
-	Assets      int64
-	PageErrors  int64
-	AssetErrors int64
-	Skipped     int64
+	Pages        int64
+	PagePaths    int64
+	PagesLinked  int64
+	Assets       int64
+	AssetSkipped int64
+	PageErrors   int64
+	AssetErrors  int64
+	Skipped      int64
 }
 
 func (s *stats) snapshot() Progress {
 	return Progress{
-		Pages:       s.pages.Load(),
-		Assets:      s.assets.Load(),
-		PageErrors:  s.pageErrors.Load(),
-		AssetErrors: s.assetErrors.Load(),
-		Skipped:     s.skipped.Load(),
+		Pages:        s.pages.Load(),
+		PagePaths:    s.pagePaths.Load(),
+		PagesLinked:  s.pagesLinked.Load(),
+		Assets:       s.assets.Load(),
+		AssetSkipped: s.assetSkipped.Load(),
+		PageErrors:   s.pageErrors.Load(),
+		AssetErrors:  s.assetErrors.Load(),
+		Skipped:      s.skipped.Load(),
 	}
 }
 
@@ -34,4 +106,6 @@ func (s *stats) snapshot() Progress {
 type Result struct {
 	Progress
 	OutDir string
+	// Failures is a capped sample of what went wrong, for the final report.
+	Failures []Failure
 }

clone/wantasset_test.go

@@ -0,0 +1,70 @@
+package clone
+
+import (
+	"testing"
+
+	"github.com/tamnd/kage/urlx"
+)
+
+// TestWantAsset checks the two URL-only skip rules a page worker applies before
+// downloading an asset: off-domain hosts and bulk-media extensions are left on
+// the live web, while the seed's own images, fonts, and stylesheets localize.
+func TestWantAsset(t *testing.T) {
+	seed, _ := urlx.ParseSeed("https://developer.apple.com/")
+	c := New(seed, DefaultConfig(), nil)
+
+	cases := []struct {
+		u    string
+		want bool
+	}{
+		// Same registrable domain: localize.
+		{"https://developer.apple.com/css/main.css", true},
+		{"https://www.apple.com/img/logo.png", true},
+		{"https://images.apple.com/fonts/sf.woff2", true},
+		// Bulk media, installers, archives, and PDFs: leave remote.
+		{"https://developer.apple.com/videos/wwdc.mp4", false},
+		{"https://developer.apple.com/downloads/Xcode.dmg", false},
+		{"https://developer.apple.com/bundle.zip", false},
+		{"https://developer.apple.com/guide.pdf", false},
+		{"https://developer.apple.com/clip.MP4", false}, // case-insensitive
+		// Off-domain hosts: leave remote even for a normal image.
+		{"https://cdn-apple.com/img/x.png", false},
+		{"https://ec.europa.eu/banner.png", false},
+		{"https://mmbiz.qpic.cn/x.jpg", false},
+	}
+	for _, tc := range cases {
+		u := mustURL(t, tc.u)
+		if got := c.wantAsset(u); got != tc.want {
+			t.Errorf("wantAsset(%q) = %v, want %v", tc.u, got, tc.want)
+		}
+	}
+}
+
+// TestWantAssetAllHosts checks that turning the domain scope off localizes a
+// third-party asset, while the media extension rule still applies.
+func TestWantAssetAllHosts(t *testing.T) {
+	seed, _ := urlx.ParseSeed("https://developer.apple.com/")
+	cfg := DefaultConfig()
+	cfg.AssetSameDomain = false
+	c := New(seed, cfg, nil)
+
+	if !c.wantAsset(mustURL(t, "https://cdn-apple.com/img/x.png")) {
+		t.Error("with AssetSameDomain off, an off-domain image should localize")
+	}
+	if c.wantAsset(mustURL(t, "https://cdn-apple.com/video/x.mp4")) {
+		t.Error("a media file should still be skipped regardless of host scope")
+	}
+}
+
+// TestWantAssetKeepMedia checks that an empty skip set (the --keep-media case)
+// localizes media too.
+func TestWantAssetKeepMedia(t *testing.T) {
+	seed, _ := urlx.ParseSeed("https://developer.apple.com/")
+	cfg := DefaultConfig()
+	cfg.SkipAssetExts = map[string]bool{}
+	c := New(seed, cfg, nil)
+
+	if !c.wantAsset(mustURL(t, "https://developer.apple.com/videos/wwdc.mp4")) {
+		t.Error("with an empty skip set, an on-domain video should localize")
+	}
+}

cmd/kage/main.go

@@ -9,9 +9,15 @@ import (
 	"os/signal"
 
 	"github.com/tamnd/kage/cli"
+	"github.com/tamnd/kage/viewer"
 )
 
 func main() {
+	// Pin the main goroutine to the process's initial OS thread before anything
+	// else. In the webview build the native window must be driven from that
+	// thread; in the default build this is a harmless no-op.
+	viewer.LockMainThread()
+
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer stop()
 	os.Exit(cli.Execute(ctx))

dataset/dataset.go

@@ -0,0 +1,307 @@
+// Package dataset converts between a packed ZIM archive and a columnar Parquet
+// file. The Parquet form is a flat table with one row per archive entry and
+// clear columns (url, mime, title, content, extracted text, redirect target),
+// which is the shape a dataset host such as Hugging Face expects. The conversion
+// is lossless: every entry, its metadata, and the main page survive a ZIM ->
+// Parquet -> ZIM round trip, so the table doubles as an archival representation,
+// not just an export.
+package dataset
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/parquet-go/parquet-go"
+	"golang.org/x/net/html"
+
+	"github.com/tamnd/kage/zim"
+)
+
+// Row is one archive entry as a Parquet record. A content or metadata entry
+// carries its bytes in Content, its type in Mime, and, for HTML, its visible
+// text in Text. A redirect sets IsRedirect and names its destination in
+// RedirectTarget ("<namespace>/<url>", e.g. "C/index.html"), with Content empty.
+//
+// The leading columns (doc_id, url, host, crawl_date, the length pair, text)
+// follow the field names the open-index/open-markdown dataset uses, so a kage
+// export drops into the same tooling other web-crawl datasets on Hugging Face
+// are read with. The trailing columns (namespace, the redirect pair, content)
+// are kage's own: they carry the raw bytes and the ZIM structure that make the
+// table a lossless, reversible copy of the archive rather than a one-way export.
+//
+// Namespace is the single-letter ZIM namespace: "C" for pages and assets, "M"
+// for archive metadata (Title, Description, Language, ...), "W" for well-known
+// entries such as the main-page pointer. Keeping every namespace as a row is
+// what makes the table reversible.
+type Row struct {
+	DocID          string `parquet:"doc_id,dict"`
+	URL            string `parquet:"url"`
+	Host           string `parquet:"host,dict"`
+	Title          string `parquet:"title"`
+	Mime           string `parquet:"mime,dict"`
+	CrawlDate      string `parquet:"crawl_date,dict"`
+	ContentLength  int64  `parquet:"content_length"`
+	TextLength     int64  `parquet:"text_length"`
+	Text           string `parquet:"text"`
+	Namespace      string `parquet:"namespace,dict"`
+	IsRedirect     bool   `parquet:"is_redirect"`
+	RedirectTarget string `parquet:"redirect_target"`
+	Content        []byte `parquet:"content"`
+}
+
+// docNamespace is the UUID v5 namespace for kage doc_id values: the standard
+// URL namespace, matching how open-markdown derives a deterministic id from a
+// page's canonical URL.
+var docNamespace = uuid.NameSpaceURL
+
+// docID returns the deterministic UUID v5 an entry gets in the doc_id column,
+// derived from its host and url so the same page always hashes to the same id
+// across exports. Entries with no host (metadata, well-known) fall back to the
+// namespaced url, which is still stable.
+func docID(host, namespace, url string) string {
+	name := namespace + "/" + url
+	if host != "" {
+		name = host + "/" + url
+	}
+	return uuid.NewSHA1(docNamespace, []byte(name)).String()
+}
+
+// Stats summarises a conversion for the CLI to report.
+type Stats struct {
+	Rows         int64 // total entries written
+	Redirects    int64 // of those, redirects
+	ContentBytes int64 // sum of stored content bytes (uncompressed)
+}
+
+// writeBatch bounds how many rows are buffered before a write to the Parquet
+// writer. It only paces the calls; the writer manages its own row groups.
+const writeBatch = 256
+
+// ZIMToParquet reads the ZIM at zimPath and writes a Parquet table to outPath,
+// one row per entry. The archive's main page is recorded both as its W/mainPage
+// redirect row and as file-level metadata, and a short generator/source line is
+// attached so the dataset is self-describing. version is kage's version string.
+func ZIMToParquet(zimPath, outPath, version string) (Stats, error) {
+	r, err := zim.Open(zimPath)
+	if err != nil {
+		return Stats{}, err
+	}
+	defer func() { _ = r.Close() }()
+
+	f, err := os.Create(outPath)
+	if err != nil {
+		return Stats{}, err
+	}
+	bw := bufio.NewWriter(f)
+	pw := parquet.NewGenericWriter[Row](bw, parquet.Compression(&parquet.Zstd))
+
+	host := metaValue(r, "Source")
+	if host == "" {
+		host = metaValue(r, "Name")
+	}
+	host = strings.ToLower(host)
+	crawlDate := metaValue(r, "Date")
+
+	if ns, u, ok := r.MainPageRef(); ok {
+		pw.SetKeyValueMetadata("kage.main_page", string(ns)+"/"+u)
+	}
+	pw.SetKeyValueMetadata("kage.generator", strings.TrimSpace("kage "+version))
+	pw.SetKeyValueMetadata("kage.source", filepath.Base(zimPath))
+	if host != "" {
+		pw.SetKeyValueMetadata("kage.host", host)
+	}
+
+	var st Stats
+	count := r.Count()
+	batch := make([]Row, 0, writeBatch)
+	flush := func() error {
+		if len(batch) == 0 {
+			return nil
+		}
+		if _, err := pw.Write(batch); err != nil {
+			return err
+		}
+		batch = batch[:0]
+		return nil
+	}
+
+	for i := uint32(0); i < count; i++ {
+		e, err := r.EntryAt(i)
+		if err != nil {
+			return st, fmt.Errorf("read entry %d: %w", i, err)
+		}
+		row := Row{
+			Namespace: string(e.Namespace),
+			URL:       e.URL,
+			Title:     e.Title,
+			Host:      host,
+			CrawlDate: crawlDate,
+			DocID:     docID(host, string(e.Namespace), e.URL),
+		}
+		if e.Redirect {
+			row.IsRedirect = true
+			row.RedirectTarget = string(e.RedirectNamespace) + "/" + e.RedirectURL
+			st.Redirects++
+		} else {
+			row.Mime = e.MimeType
+			row.Content = e.Data
+			row.ContentLength = int64(len(e.Data))
+			st.ContentBytes += int64(len(e.Data))
+			if e.MimeType == "text/html" {
+				row.Text = htmlText(e.Data)
+				row.TextLength = int64(len(row.Text))
+			}
+		}
+		st.Rows++
+		batch = append(batch, row)
+		if len(batch) == cap(batch) {
+			if err := flush(); err != nil {
+				return st, err
+			}
+		}
+	}
+	if err := flush(); err != nil {
+		return st, err
+	}
+	if err := pw.Close(); err != nil {
+		return st, err
+	}
+	if err := bw.Flush(); err != nil {
+		return st, err
+	}
+	return st, f.Close()
+}
+
+// ParquetToZIM reads the Parquet table at parquetPath and writes the ZIM archive
+// it describes to outPath, reproducing every entry, its metadata, and the main
+// page. version is unused for now; it is accepted so the signature matches its
+// sibling and can record provenance later.
+func ParquetToZIM(parquetPath, outPath, _ string) (Stats, error) {
+	f, err := os.Open(parquetPath)
+	if err != nil {
+		return Stats{}, err
+	}
+	defer func() { _ = f.Close() }()
+
+	pr := parquet.NewGenericReader[Row](f)
+	defer func() { _ = pr.Close() }()
+
+	w := zim.NewWriter()
+	var st Stats
+	var mainNS byte
+	var mainURL string
+	haveMain := false
+
+	buf := make([]Row, writeBatch)
+	for {
+		n, readErr := pr.Read(buf)
+		for i := 0; i < n; i++ {
+			row := buf[i]
+			ns := namespaceByte(row.Namespace)
+			if row.IsRedirect {
+				tns, turl := splitTarget(row.RedirectTarget)
+				w.AddRedirect(ns, row.URL, row.Title, tns, turl)
+				if ns == zim.NamespaceWellKnown && row.URL == "mainPage" {
+					mainNS, mainURL, haveMain = tns, turl, true
+				}
+				st.Redirects++
+			} else {
+				w.AddContent(ns, row.URL, row.Title, row.Mime, row.Content)
+				st.ContentBytes += int64(len(row.Content))
+			}
+			st.Rows++
+		}
+		if readErr == io.EOF {
+			break
+		}
+		if readErr != nil {
+			return st, readErr
+		}
+		if n == 0 {
+			break
+		}
+	}
+	if haveMain {
+		w.SetMainPage(mainNS, mainURL)
+	}
+
+	out, err := os.Create(outPath)
+	if err != nil {
+		return st, err
+	}
+	obw := bufio.NewWriter(out)
+	if _, err := w.WriteTo(obw); err != nil {
+		_ = out.Close()
+		return st, err
+	}
+	if err := obw.Flush(); err != nil {
+		_ = out.Close()
+		return st, err
+	}
+	return st, out.Close()
+}
+
+// metaValue reads an M/ metadata entry as a string, returning "" when it is
+// absent. It lets the exporter fill the host and crawl_date columns from the
+// archive's own metadata.
+func metaValue(r *zim.Reader, name string) string {
+	b, err := r.Get(zim.NamespaceMetadata, name)
+	if err != nil {
+		return ""
+	}
+	return string(b.Data)
+}
+
+// namespaceByte turns a one-letter namespace column back into the ZIM byte,
+// defaulting to the content namespace for an unexpectedly empty value.
+func namespaceByte(s string) byte {
+	if s == "" {
+		return zim.NamespaceContent
+	}
+	return s[0]
+}
+
+// splitTarget parses a "<namespace>/<url>" redirect target back into its parts.
+// The namespace is a single byte, so the url is everything after the first two
+// characters; a malformed value yields the content namespace and the raw string.
+func splitTarget(s string) (byte, string) {
+	if len(s) >= 2 && s[1] == '/' {
+		return s[0], s[2:]
+	}
+	return zim.NamespaceContent, s
+}
+
+// htmlText extracts the visible text of an HTML document: the concatenated text
+// nodes outside script, style, and noscript, with runs of whitespace collapsed
+// to single spaces. It is a derived convenience column for dataset consumers and
+// plays no part in the round trip, which reconstructs pages from Content.
+func htmlText(data []byte) string {
+	doc, err := html.Parse(strings.NewReader(string(data)))
+	if err != nil {
+		return ""
+	}
+	var b strings.Builder
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			switch n.Data {
+			case "script", "style", "noscript":
+				return
+			}
+		}
+		if n.Type == html.TextNode {
+			b.WriteString(n.Data)
+			b.WriteByte(' ')
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+	return strings.Join(strings.Fields(b.String()), " ")
+}

dataset/dataset_test.go

@@ -0,0 +1,115 @@
+package dataset
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/tamnd/kage/zim"
+)
+
+// buildZIM writes a small archive with a page, an asset, metadata, and a
+// main-page redirect to a temp file, and returns its path.
+func buildZIM(t *testing.T) string {
+	t.Helper()
+	w := zim.NewWriter()
+	w.AddContent(zim.NamespaceContent, "index.html", "Home",
+		"text/html", []byte("<html><head><title>Home</title></head><body><script>ignore()</script><h1>Hello</h1><p>World</p></body></html>"))
+	w.AddContent(zim.NamespaceContent, "logo.png", "", "image/png", []byte{0x89, 'P', 'N', 'G', 1, 2, 3})
+	w.AddMetadata("Title", "Test Site")
+	w.AddMetadata("Language", "eng")
+	w.SetMainPage(zim.NamespaceContent, "index.html")
+	w.AddRedirect(zim.NamespaceWellKnown, "mainPage", "", zim.NamespaceContent, "index.html")
+
+	path := filepath.Join(t.TempDir(), "site.zim")
+	f, err := os.Create(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := w.WriteTo(f); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+func TestZIMParquetRoundTrip(t *testing.T) {
+	src := buildZIM(t)
+	dir := t.TempDir()
+	pq := filepath.Join(dir, "site.parquet")
+	dst := filepath.Join(dir, "site2.zim")
+
+	exp, err := ZIMToParquet(src, pq, "test")
+	if err != nil {
+		t.Fatalf("ZIMToParquet: %v", err)
+	}
+	if exp.Rows == 0 {
+		t.Fatal("exported zero rows")
+	}
+	if exp.Redirects == 0 {
+		t.Fatal("expected the mainPage redirect to be exported")
+	}
+
+	imp, err := ParquetToZIM(pq, dst, "test")
+	if err != nil {
+		t.Fatalf("ParquetToZIM: %v", err)
+	}
+	if imp.Rows != exp.Rows {
+		t.Fatalf("row count changed across round trip: exported %d, imported %d", exp.Rows, imp.Rows)
+	}
+	if imp.Redirects != exp.Redirects {
+		t.Fatalf("redirect count changed: exported %d, imported %d", exp.Redirects, imp.Redirects)
+	}
+
+	// The rebuilt archive must serve the same content and resolve its main page.
+	r, err := zim.Open(dst)
+	if err != nil {
+		t.Fatalf("open rebuilt zim: %v", err)
+	}
+	defer func() { _ = r.Close() }()
+
+	home, err := r.Get(zim.NamespaceContent, "index.html")
+	if err != nil {
+		t.Fatalf("get index.html: %v", err)
+	}
+	if got := string(home.Data); got != "<html><head><title>Home</title></head><body><script>ignore()</script><h1>Hello</h1><p>World</p></body></html>" {
+		t.Fatalf("page content changed: %q", got)
+	}
+	if home.MimeType != "text/html" {
+		t.Fatalf("page mime changed: %q", home.MimeType)
+	}
+
+	logo, err := r.Get(zim.NamespaceContent, "logo.png")
+	if err != nil {
+		t.Fatalf("get logo.png: %v", err)
+	}
+	if string(logo.Data) != string([]byte{0x89, 'P', 'N', 'G', 1, 2, 3}) {
+		t.Fatal("asset bytes changed across round trip")
+	}
+
+	title, err := r.Get(zim.NamespaceMetadata, "Title")
+	if err != nil {
+		t.Fatalf("get M/Title: %v", err)
+	}
+	if string(title.Data) != "Test Site" {
+		t.Fatalf("metadata changed: %q", string(title.Data))
+	}
+
+	main, err := r.MainPage()
+	if err != nil {
+		t.Fatalf("main page not set after round trip: %v", err)
+	}
+	if main.URL != "index.html" {
+		t.Fatalf("main page url changed: %q", main.URL)
+	}
+}
+
+func TestHTMLTextStripsScript(t *testing.T) {
+	got := htmlText([]byte("<html><body><script>secret()</script><style>.x{}</style><h1>Visible</h1>  <p>Text\nhere</p></body></html>"))
+	want := "Visible Text here"
+	if got != want {
+		t.Fatalf("htmlText = %q, want %q", got, want)
+	}
+}

docs/content/_index.md

@@ -7,35 +7,32 @@ heroPrimaryURL: "/getting-started/quick-start/"
 heroPrimaryText: "Get started"
 ---
 
-Saving a page with "Save As" gives you a copy that still phones home, still runs
-analytics, and often renders blank because the markup is built by JavaScript at
-runtime. kage (影, "shadow") takes the opposite approach: it drives a real
-browser, captures the page the way a human would have seen it, then makes it
-inert.
+Saving a page with "Save As" gives you a copy that still phones home, still runs analytics, and often renders blank because the markup is built by JavaScript at runtime. kage (影, "shadow") takes the opposite approach: it drives a real browser, captures the page the way a human would have seen it, then makes it inert.
+
+Say you want Paul Graham's essays on a laptop with no wifi. One command mirrors the site; a second serves it back offline:
 
 ```bash
-kage clone example.com
-kage serve kage-out/example.com
+kage clone paulgraham.com
+kage serve $HOME/data/kage/paulgraham.com
 ```
 
+![kage cloning paulgraham.com, packing it into one file, and serving it back offline](/demo.gif)
+
 ## What it does
 
-- **Renders first, saves second.** Each page goes through real headless Chrome,
-  so a page whose content is assembled by JavaScript is captured fully, not as
-  an empty shell.
-- **Strips every script.** Once the DOM is captured, kage removes all `<script>`
-  tags, every `on*` event handler, and any `javascript:` URL. The saved page
-  makes no network calls and runs no code.
-- **Keeps the layout.** Stylesheets, images, fonts, and media are downloaded and
-  rewritten to relative local paths, so the offline copy looks like the original.
-- **Stays browsable.** In-scope links are rewritten to point at the other saved
-  pages, so you can click around the mirror exactly as you would the live site.
+- **Renders first, saves second.** Each page goes through real headless Chrome, so a page whose content is assembled by JavaScript is captured fully, not as an empty shell.
+- **Strips every script.** Once the DOM is captured, kage removes all `<script>` tags, every `on*` event handler, and any `javascript:` URL. The saved page makes no network calls and runs no code.
+- **Keeps the layout.** Stylesheets, images, fonts, and media are downloaded and rewritten to relative local paths, so the offline copy looks like the original.
+- **Stays browsable.** In-scope links are rewritten to point at the other saved pages, so you can click around the mirror exactly as you would the live site.
+- **Packs into one file.** Collapse a mirror into a single [ZIM archive](/guides/packing-a-mirror/), the open format Kiwix uses, or a self-contained binary that serves the site when run.
+
+Build kage with the `webview` tag and a packed binary opens in its own window instead of a browser tab, so an offline mirror feels like a real app:
+
+![paulgraham.com served offline in a native kage window](/webview.png)
 
 ## Where to go next
 
-- New here? Start with the [introduction](/getting-started/introduction/), then
-  the [quick start](/getting-started/quick-start/).
+- New here? Start with the [introduction](/getting-started/introduction/), then the [quick start](/getting-started/quick-start/).
 - Want to install it? See [installation](/getting-started/installation/).
-- Looking for a specific task? The [guides](/guides/) cover scoping a crawl,
-  serving a mirror, and resuming an interrupted run.
+- Looking for a specific task? The [guides](/guides/) cover scoping a crawl, serving a mirror, resuming an interrupted run, and [packing a mirror](/guides/packing-a-mirror/) into one file or a self-contained viewer.
 - Need every flag? The [CLI reference](/reference/cli/) is the full surface.

docs/content/getting-started/introduction.md

@@ -4,21 +4,20 @@ description: "Why kage renders before it saves, and what it means to strip the J
 weight: 10
 ---
 
-A normal website is not a document; it is a program. The HTML the server sends
-is often a near-empty shell, and the page you actually see is assembled in your
-browser by JavaScript: fetching data, building the DOM, wiring up handlers. That
-is why "Save As" so often fails. You get the shell, not the page, and whatever
-you do get still runs trackers and phones home when you open it.
+A normal website is not a document; it is a program. The HTML the server sends is often a near-empty shell, and the page you actually see is assembled in your browser by JavaScript: fetching data, building the DOM, wiring up handlers. That is why "Save As" so often fails. You get the shell, not the page, and whatever you do get still runs trackers and phones home when you open it.
+
+Say you want to keep Paul Graham's essays. Hand the site to "Save As" and you get a brittle copy that may still call out to scripts that no longer exist. Hand it to kage and you get the essays as they look in a browser, frozen and inert:
+
+```bash
+kage clone paulgraham.com
+kage serve $HOME/data/kage/paulgraham.com
+```
 
 kage treats a clone as three steps in order.
 
 ## 1. Render
 
-Every page is loaded in a real headless Chrome through the DevTools protocol.
-kage navigates to the URL, waits for the network to go quiet, optionally scrolls
-to trigger lazy-loaded images, and then serialises the **final** DOM, the markup
-that exists after the page's JavaScript has finished building it. This is the
-same thing you would see if you opened the page and chose "Inspect".
+Every page is loaded in a real headless Chrome through the DevTools protocol. kage navigates to the URL, waits for the network to go quiet, optionally scrolls to trigger lazy-loaded images, and then serialises the **final** DOM, the markup that exists after the page's JavaScript has finished building it. This is the same thing you would see if you opened the page and chose "Inspect".
 
 ## 2. Strip
 
@@ -27,27 +26,22 @@ From that captured DOM, kage removes everything executable:
 - every `<script>` tag, inline or external;
 - every `on*` event handler attribute (`onclick`, `onload`, and the rest);
 - every `javascript:` URL;
-- `<meta http-equiv="refresh">` redirects and dead resource hints like
-  `<link rel="preload" as="script">`.
+- `<meta http-equiv="refresh">` redirects and dead resource hints like `<link rel="preload" as="script">`.
 
-What remains is inert. The saved page makes no network calls, runs no code, and
-tracks nothing.
+What remains is inert. The saved page makes no network calls, runs no code, and tracks nothing.
 
 ## 3. Localise
 
-A page with no working CSS or images is not much of a clone, so kage keeps the
-parts that define how it looks. It downloads every stylesheet, image, font, and
-media file, rewrites the references in the HTML and inside the CSS (`url()` and
-`@import`) to relative local paths, and rewrites in-scope page links to point at
-the other saved pages. The mirror is fully self-contained: you can move the
-folder anywhere, open it with no network, and click around.
+A page with no working CSS or images is not much of a clone, so kage keeps the parts that define how it looks. It downloads every stylesheet, image, font, and media file, rewrites the references in the HTML and inside the CSS (`url()` and `@import`) to relative local paths, and rewrites in-scope page links to point at the other saved pages. The mirror is fully self-contained: you can move the folder anywhere, open it with no network, and click around.
 
 ## The shape of a clone
 
-kage crawls breadth-first from a seed URL, staying within the seed's host (and
-optionally its subdomains). It is polite by default: it honours `robots.txt` and
-seeds itself from `sitemap.xml`. Output lands in `kage-out/<host>/`, with pages
-as `<path>/index.html` and assets under a reserved `_kage/` directory alongside
-the crawl state that powers `--resume`.
+kage crawls breadth-first from a seed URL, staying within the seed's host (and optionally its subdomains). It is polite by default: it honours `robots.txt` and seeds itself from `sitemap.xml`. Output lands in `$HOME/data/kage/paulgraham.com/`, with pages as `<path>/index.html` and assets under a reserved `_kage/` directory alongside the crawl state that powers resuming.
+
+## Then what?
+
+A folder is the starting point, not the end. Once you have a mirror you can [pack it](/guides/packing-a-mirror/) into a single ZIM file, the open offline-archive format Kiwix uses, so the whole site travels as one file that any ZIM reader can open. You can also pack it into a self-contained binary, or a double-click desktop app with the site's favicon as its icon. Or build kage with the `webview` tag and a packed binary opens the site in its own native window instead of a browser tab:
+
+![paulgraham.com served offline in a native kage window](/webview.png)
 
 Next: [install kage](/getting-started/installation/).

docs/content/getting-started/quick-start.md

@@ -19,15 +19,15 @@ errors as it goes; the final summary tells you where the mirror landed.
 
 ```
 kage cloning https://example.com
-done kage-out/example.com
+done $HOME/data/kage/example.com
   pages 12   assets 38
-  open kage serve kage-out/example.com
+  open kage serve $HOME/data/kage/example.com
 ```
 
 ## 2. Look at what landed
 
 ```bash
-ls kage-out/example.com
+ls $HOME/data/kage/example.com
 ```
 
 ```
@@ -46,7 +46,7 @@ serve` runs a local static server so everything resolves exactly as it would
 live:
 
 ```bash
-kage serve kage-out/example.com
+kage serve $HOME/data/kage/example.com
 # open http://127.0.0.1:8800
 ```
 

docs/content/guides/packing-a-mirror.md

@@ -0,0 +1,154 @@
+---
+title: "Packing a mirror"
+description: "Turn a cloned folder into one ZIM file or a self-contained offline viewer with kage pack."
+weight: 30
+---
+
+A clone is a folder of files, which is easy to browse but awkward to move around: copying thousands of small files is slow, and handing someone a directory is less tidy than handing them one file. `kage pack` collapses a mirror into a single distributable artifact, and you choose the shape: an open ZIM archive, or a self-contained executable that serves the site offline when run.
+
+The two examples below assume you have already cloned a site, for instance Paul Graham's essays:
+
+```bash
+kage clone paulgraham.com
+```
+
+## A single ZIM file
+
+ZIM is the open, single-file offline-archive format Kiwix uses. `kage pack` writes one from a cloned host directory:
+
+```bash
+kage pack paulgraham.com
+```
+
+```
+packed paulgraham.com.zim
+  size 4.2 MiB
+  open kage open paulgraham.com.zim
+```
+
+The whole mirror, pages and assets, lives in that one file. Text is zstd compressed; already-compressed media (images, fonts, video) is stored as-is. Packing the same mirror twice produces a byte-identical file, so a ZIM is safe to checksum, diff, and cache.
+
+If you cloned with the default output directory, you can pass a bare host name and kage finds the mirror for you. That is why `kage pack paulgraham.com` works straight after `kage clone paulgraham.com`; pass a full path if your mirror lives somewhere else.
+
+### What ZIM is, and using it with Kiwix
+
+ZIM is built for exactly this job: a whole website (or a whole Wikipedia) squeezed into one compressed, indexed, read-only file. It is the format behind [Kiwix](https://kiwix.org), the offline-content project people use to carry Wikipedia, Stack Overflow, and Project Gutenberg onto boats, into classrooms with no internet, and onto a phone for a long flight. Because the format is a documented standard rather than a kage invention, a `paulgraham.com.zim` you make today still opens in any ZIM reader years from now.
+
+So you are not locked into kage. `kage open` is the read side and the quickest way back in, but the same file works across the wider Kiwix ecosystem:
+
+```bash
+kage open paulgraham.com.zim            # read it back with kage
+kiwix-serve paulgraham.com.zim          # or serve it with Kiwix at http://localhost
+```
+
+You can also double-click the file in the [Kiwix desktop app](https://kiwix.org/en/applications/), or load it on Kiwix for Android or iOS to read your mirror on your phone. kage writes the metadata the format and `zimcheck` treat as mandatory, including the title, description, and the favicon Kiwix shows as the book icon in its library, so the archive shows up properly rather than as an untitled, iconless entry. One caveat: kage does not write the full-text search index that Kiwix's own packs ship with, so browsing works everywhere while in-reader search is limited.
+
+## A self-contained binary
+
+`--format binary` appends the ZIM to a copy of kage, producing one executable that *is* the site. Run it and it serves the mirror on a free local port and opens your browser; it ignores its arguments, because the binary is the site, not the kage CLI.
+
+```bash
+kage pack paulgraham.com --format binary -o paulgraham
+```
+
+```
+packed paulgraham
+  size 21.9 MiB
+  run ./paulgraham to view the site offline
+```
+
+```bash
+./paulgraham
+```
+
+```
+serving offline site at http://127.0.0.1:52431  (Ctrl-C to stop)
+```
+
+The binary carries a full kage, so it is tens of megabytes regardless of site size; the trade is that the recipient needs nothing installed, not even kage, not even a ZIM reader.
+
+### A native window instead of a browser
+
+By default the viewer opens the system browser, which means a tab with an address bar and your other tabs alongside. Build kage with the `webview` tag and it opens the site in its own native window instead, backed by the operating system's WebView (WKWebView on macOS, WebView2 on Windows, WebKitGTK on Linux), so a packed binary feels like a standalone app:
+
+```bash
+CGO_ENABLED=1 go build -tags webview -o kage ./cmd/kage
+kage pack paulgraham.com --format binary --base kage -o paulgraham
+./paulgraham   # opens a window, no browser
+```
+
+![paulgraham.com served offline in a native kage window](/webview.png)
+
+The window title comes from the archive's title. This build needs cgo and links the platform WebView, so it is opt-in and kept out of the default `CGO_ENABLED=0` release; the prebuilt binaries open the browser. `kage open` honours the same tag: built with `-tags webview` it shows the ZIM in a native window too.
+
+### Build a viewer for another platform
+
+The appended archive is platform-independent; only the base executable carries the architecture. Point `--base` at a kage binary built for another OS (download one from a kage release; every platform ships one) to produce a viewer for that platform from your own machine. kage reads the base's executable header to detect the target OS, so a Windows viewer automatically gets a `.exe` name and the run hint names the right platform:
+
+```bash
+# From macOS, build a Windows viewer
+kage pack paulgraham.com --format binary --base kage-windows-amd64.exe
+# -> paulgraham.exe
+```
+
+### macOS note
+
+A binary you built or downloaded may be quarantined by Gatekeeper on first run. kage prints the exact command to clear it:
+
+```bash
+xattr -d com.apple.quarantine ./paulgraham
+```
+
+## A double-click app
+
+The self-contained binary is perfect from a terminal, but double-clicking it in a file manager is less tidy: on macOS Finder opens a Terminal window behind the site, and on Windows a console flashes alongside it. Add `--app` and kage wraps the same viewer in a real desktop app, so a double-click just opens the mirror with no terminal in sight, using the site's own favicon as the icon.
+
+On macOS it writes a standard `.app` bundle:
+
+```bash
+kage pack paulgraham.com --app
+```
+
+```
+packed paulgraham.app
+  size 13.5 MiB
+  icon paulgraham.com/apple-touch-icon.png
+  double-click paulgraham.app to open the site offline
+```
+
+The bundle holds the packed viewer under `Contents/MacOS`, an `Info.plist` describing the app, and the icon converted to `Contents/Resources/icon.icns`. Double-click it in Finder, or run `open paulgraham.app`, and the site comes up with no console attached.
+
+On Linux, point `--base` at a Linux kage and you get an [AppImage](https://appimage.org)-style `.AppDir`: the viewer as `AppRun`, a `.desktop` launcher with `Terminal=false`, and the icon as a PNG. When [`appimagetool`](https://github.com/AppImage/appimagetool) is on your `PATH`, kage runs it for you and turns the directory into one double-clickable `.AppImage`; otherwise it leaves the `.AppDir` ready for any AppImage tool.
+
+```bash
+kage pack paulgraham.com --app --base kage-linux-amd64   # -> paulgraham.AppDir (+ .AppImage)
+```
+
+kage picks the icon by digging through the mirror for the site's favicon. It prefers a large `apple-touch-icon.png` and falls back to `favicon.png` or a PNG-based `favicon.ico`; if a site only ships a legacy BMP `.ico` the bundle is built without a custom icon rather than with a mangled one. Override the choice with `--icon path/to/image.png`.
+
+For the full "it's an app" effect, pair `--app` with a `webview` base so the double-click opens a native window instead of the system browser:
+
+```bash
+make build-webview
+kage pack paulgraham.com --app --base bin/kage
+```
+
+Windows needs no bundle, because there a single `.exe` already is the app. What it needs is to lose the console window. A normal build is console-attached (handy for the CLI, since that is where clone progress prints), so the release ships a second Windows binary linked for the GUI subsystem in `kage_<version>_windows-gui_<arch>.zip`. Pack a viewer onto that base and double-clicking the result opens the site with no console behind it:
+
+```bash
+kage pack paulgraham.com --format binary --base kage-windows-gui-amd64.exe   # -> paulgraham.exe
+```
+
+## Metadata and options
+
+```bash
+kage pack paulgraham.com \
+  --title "Paul Graham, offline" \
+  --description "A snapshot taken for archival" \
+  --language eng \
+  --date 2026-06-14
+```
+
+`--title` defaults to the main page's `<title>`, then the host name. `--description` defaults to a line derived from the host, since a description is mandatory metadata. `--date` defaults to today; pass a fixed value for a fully reproducible file. `--no-compress` stores every cluster raw, which packs fastest and lets a reader without zstd open the result. `-o/--out` overrides the output path for either format.
+
+Beyond these, kage fills in the rest of the mandatory metadata on its own: a unique `Name`, the `Language`, and an `Illustrator_48x48@1` icon. The icon is the site's favicon dug out of the mirror (an `apple-touch-icon.png`, `favicon.png`, or a PNG-based `favicon.ico`) and rescaled to a 48x48 PNG, the same discovery `--app` uses for the desktop icon. A site that ships only a legacy BMP `.ico` or no icon at all is packed without one rather than with a broken image.

docs/content/guides/resuming-a-run.md

@@ -37,7 +37,7 @@ existing host folder first with `--force`:
 kage clone example.com --force
 ```
 
-This removes `kage-out/example.com/` before crawling, so nothing from a prior run
+This removes `$HOME/data/kage/example.com/` before crawling, so nothing from a prior run
 carries over.
 
 To run without reading or writing any resume state at all, for a strictly

docs/content/guides/serving-a-mirror.md

@@ -12,11 +12,11 @@ when served from the root of a host. `kage serve` gives you that root.
 ## Serve a clone
 
 ```bash
-kage serve kage-out/example.com
+kage serve $HOME/data/kage/example.com
 ```
 
 ```
-kage serve /…/kage-out/example.com
+kage serve $HOME/data/kage/example.com
   open http://127.0.0.1:8800
   press Ctrl-C to stop
 ```
@@ -31,10 +31,10 @@ By default kage serves on `127.0.0.1:8800`. Change it with `--addr`:
 
 ```bash
 # A different port
-kage serve kage-out/example.com --addr 127.0.0.1:9000
+kage serve $HOME/data/kage/example.com --addr 127.0.0.1:9000
 
 # Reachable from other machines on your network (be deliberate about this)
-kage serve kage-out/example.com --addr 0.0.0.0:8800
+kage serve $HOME/data/kage/example.com --addr 0.0.0.0:8800
 ```
 
 ## Serve the current directory
@@ -43,6 +43,6 @@ With no argument, `kage serve` serves the current directory, which is handy from
 inside an output folder:
 
 ```bash
-cd kage-out/example.com
+cd $HOME/data/kage/example.com
 kage serve
 ```

docs/content/reference/cli.md

@@ -8,8 +8,9 @@ weight: 10
 kage [command] [flags]
 ```
 
-Two commands: `clone` fetches a site into an offline folder, `serve` previews
-one. Run `kage <command> --help` for the canonical, up-to-date list.
+Four commands: `clone` fetches a site into an offline folder, `serve` previews
+one, `pack` collapses a mirror into a single file, and `open` serves a packed
+file. Run `kage <command> --help` for the canonical, up-to-date list.
 
 ## kage clone
 
@@ -68,7 +69,6 @@ images, and fonts, and writes a browsable mirror to `<out>/<host>/`.
 | `--workers` | `4` | Concurrent page render workers |
 | `--asset-workers` | `8` | Concurrent asset download workers |
 | `--browser-pages` | `4` | Chrome page-pool size |
-| `--max-asset-mb` | `25` | Skip assets larger than N MB |
 | `--timeout` | `30s` | Per-request timeout |
 | `-q, --quiet` | `false` | Suppress per-page progress lines |
 
@@ -84,3 +84,45 @@ current directory.
 | Flag | Default | Meaning |
 |------|---------|---------|
 | `-a, --addr` | `127.0.0.1:8800` | Address to listen on |
+
+## kage pack
+
+```
+kage pack <mirror-dir> [flags]
+```
+
+Packs a cloned mirror into one distributable file: an open ZIM archive, or a
+self-contained executable that serves the site offline when run. A bare host name
+is resolved against the default output directory, so `kage pack example.com`
+works right after `kage clone example.com`.
+
+| Flag | Default | Meaning |
+|------|---------|---------|
+| `--format` | `zim` | Output format: `zim` or `binary` |
+| `-o, --out` | per format | Output path; `<host>.zim` for zim, `<host>` (or `<host>.exe`) for binary |
+| `--base` | this kage | Base kage binary to append to (`--format binary`); point at another platform's binary to build a viewer for it |
+| `--app` | `false` | Wrap the viewer in a double-click desktop app (`.app` on macOS, `.AppImage`/`.AppDir` on Linux) with the site's favicon as the icon |
+| `--icon` | site favicon | Icon file for `--app`, overriding the favicon found in the mirror |
+| `--no-compress` | `false` | Store every cluster raw, no zstd |
+| `--title` | main page `<title>` | Archive title |
+| `--description` | host-derived line | Archive description (mandatory metadata, defaulted when unset) |
+| `--language` | `eng` | Archive language code |
+| `--date` | today | Archive date (`YYYY-MM-DD`); pass a fixed value for a reproducible file |
+
+## kage open
+
+```
+kage open <file.zim> [flags]
+```
+
+Serves a packed ZIM over a local HTTP server for offline reading, the read side
+of `kage pack --format zim`.
+
+| Flag | Default | Meaning |
+|------|---------|---------|
+| `-a, --addr` | `127.0.0.1:8800` | Address to listen on |
+| `--open` | `true` | Open the default browser (`--open=false` to skip) |
+
+Built with `-tags webview` (which needs cgo), `kage open` shows the archive in a
+native window instead of the browser, and `--open` no longer applies. The default
+`CGO_ENABLED=0` build uses the browser.

docs/content/reference/configuration.md

@@ -24,7 +24,7 @@ A clone of `example.com` lands under `$HOME/data/kage/example.com/` (override th
 root with `-o/--out`):
 
 ```
-kage-out/example.com/
+$HOME/data/kage/example.com/
 ├── index.html                  # the home page (/), scripts stripped
 ├── about/index.html            # /about
 ├── blog/
@@ -36,7 +36,7 @@ kage-out/example.com/
 │   │   ├── logo.png
 │   │   └── fonts/body.woff2
 │   ├── cdn.example.com/        # assets from other hosts, by host
-│   └── state.json              # visited set, for --resume
+│   └── state.json              # visited set, for resume
 └── ...
 ```
 

docs/content/reference/release-notes.md

@@ -4,24 +4,77 @@ description: "What changed in each kage release."
 weight: 40
 ---
 
-The authoritative, commit-level history lives in
-[`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the
-[releases page](https://github.com/tamnd/kage/releases). This page summarises
-each version.
+The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version.
+
+## v0.3.3
+
+A fix for Chrome saving a file to your Downloads folder mid-crawl.
+
+- **A crawl never writes to your Downloads folder.** A link with no file extension is queued as a page, so the page worker opened it in Chrome. When such a link served a binary, a zip or a CSV, Chrome saved the file to `~/Downloads`, a surprise side effect of running a clone ([#32](https://github.com/tamnd/kage/issues/32)). kage now denies Chrome-initiated downloads outright, since every asset is fetched through kage's own downloader and the browser never needs to write a file. As a second layer, kage detects a navigation whose response is not HTML and reroutes that URL to the asset downloader, where the size and media policy decides whether to localise it or leave it on the live web, instead of saving a broken page.
+
+## v0.3.2
+
+A fix for garbled text on pages that did not carry a charset of their own.
+
+- **Saved pages declare UTF-8.** kage writes every page as UTF-8, but a site that set its charset only in the HTTP `Content-Type` header, with no `<meta charset>` in the markup, lost that signal once the page became a standalone file. A reader serving the bytes without a charset fell back to its locale encoding and turned every curly quote, dash, and non-breaking space into mojibake. kage now inserts a `<meta charset="utf-8">` at the top of `<head>` when the page does not already declare one, so the page renders correctly in any reader.
+
+## v0.3.1
+
+A fix for broken styling when a packed mirror's home page is a nested page.
+
+- **`/` redirects to the main page instead of serving it in place.** A page's saved asset links are mirror-relative (`../_kage/...`), computed for that page's own location. The viewer was answering `/` with the main page's bytes directly, so the browser resolved those links against `/` and 404ed the page's CSS and images. A `developer.apple.com/documentation` mirror opened at `/` came up completely unstyled. kage now redirects `/` to the main page's canonical path, the way the archive's `W/mainPage` redirect does, so relative assets resolve correctly. Kiwix already followed that redirect, so it was never affected.
+
+## v0.3.0
+
+Leaner mirrors, and a way to publish one as a dataset. A clone now keeps the assets that make a site readable offline and leaves the bulk downloads on the live web, and a packed archive converts to a columnar table that drops straight into dataset tooling.
+
+- **Bulk downloads stay remote by default.** Video and audio, installers and disk images (`.dmg`, `.pkg`, `.exe`, `.msi`, ...), archives, and PDFs are left pointing at their live URL rather than downloaded, because they are rarely needed to read a site offline yet routinely make up most of its bytes. On a `developer.apple.com` crawl that class was 18 of 19 GB. Page-rendering assets (images, fonts, CSS) are untouched. `--keep-media` restores the old behaviour, and `--skip-ext .foo` leaves more extensions remote.
+- **Assets come only from the site's own domain by default.** Localising is scoped to the seed's registrable domain, so `developer.apple.com` still pulls from `www.apple.com` and `images.apple.com` but not a separate brand domain or an off-topic third party (an embedded tracker, an unrelated CDN). `--all-asset-hosts` downloads from any host as before.
+- **The size cap skips instead of truncating.** An asset over `--max-asset-mb` was being saved as exactly the first N MB of itself, a corrupt fragment that would never play or run. kage now checks the response size and leaves an over-cap asset out of the mirror entirely, pointing at its live URL. On the apple crawl this was about a gigabyte of half-downloaded WWDC videos and `.dmg` installers.
+- **`kage parquet export` and `import`.** A packed ZIM converts to a flat Parquet table, one row per entry with clear columns (`doc_id`, `url`, `host`, `crawl_date`, `mime`, text, content), the shape a dataset host like [Hugging Face](https://huggingface.co) expects and that DuckDB or pandas reads as is. The column names follow the [open-index/open-markdown](https://huggingface.co/datasets/open-index/open-markdown) dataset, with `doc_id` a deterministic UUID v5 of the page URL, so a kage export sits alongside other web-crawl datasets. The conversion is lossless: a ZIM round-tripped through Parquet reproduces every entry, its metadata, and the main page byte for byte.
+- **`kage pack --incremental`.** Packing keeps a small cache sidecar next to the output and reuses the compression of any cluster whose bytes have not changed since the last pack. Compressing clusters with zstd is the dominant cost of packing a large mirror, so re-packing after a small change (a `--refresh`, a handful of edited pages) only compresses what actually changed. A reused cluster is byte-for-byte what a fresh compression produces, so the archive stays deterministic.
+- **Identical pages are stored once.** When a rendered page's bytes match a page already written, kage stores it as a hard link to the first copy instead of a second full file, collapsing the duplicate content a faceted site spawns when many `?q=…`/`?page=…` URLs render the same page. The summary reports how many were deduped.
+- **Cleaner progress counting.** The live counter shows distinct URL paths as "pages" and the query-string permutations one path can spawn separately as "variants", so the number tracks the site's real size instead of being inflated by `?q=…` URLs.
+
+## v0.2.1
+
+Packed ZIM archives now carry the metadata Kiwix expects, so a mirror shows up in a ZIM reader's library with a title, a description, and an icon instead of as a blank entry.
+
+- **Mandatory metadata is always written.** Every archive now gets a `Name` and a `Description` (a line derived from the host when `--description` is not given), the two fields `zimcheck` flags as missing otherwise.
+- **The favicon becomes the book icon.** When the mirror has a usable icon (an `apple-touch-icon.png`, `favicon.png`, or a PNG-based `favicon.ico`), kage rescales it to a 48x48 PNG and stores it as `Illustrator_48x48@1`, which is the icon Kiwix shows for the archive in its library. A site with no usable icon is packed without one rather than with a broken image.
+
+## v0.2.0
+
+Double-click apps, so a packed mirror opens like a real desktop app instead of a terminal program.
+
+- **`kage pack --app`** wraps the viewer in a double-click app with the site's favicon as its icon. The flag builds on the binary format, so it composes with `--base` (including a `webview` base) and `--icon`. On macOS that is a `.app` bundle; on Linux, with a Linux `--base`, an [AppImage](https://appimage.org)-style `.AppDir` that becomes a single `.AppImage` when `appimagetool` is installed. The icon is pulled from the mirror automatically, or set with `--icon`.
+- **A GUI-subsystem Windows base** ships in the release as `kage_<version>_windows-gui_<arch>.zip`. Pack a viewer onto it with `--format binary --base` and the resulting `.exe` opens with no console window behind it.
+- **Smarter cross-platform packing.** kage reads the base binary's executable header to detect its target OS, so a Windows viewer always gets a `.exe` name and the right run hint, regardless of how the base file is named.
+
+## v0.1.2
+
+A security fix for how kage launches Chrome, clearer crawl errors, and a container image that actually runs.
+
+- **Chrome keeps its sandbox on by default.** Earlier versions launched Chrome with `--no-sandbox` on every run, which switched off the browser's main security boundary even on an ordinary desktop where the sandbox works fine ([#10](https://github.com/tamnd/kage/issues/10)). The sandbox now stays on, and is dropped only where it genuinely cannot start: inside a container (detected from `IN_DOCKER` or `/.dockerenv`) or when running as root. Whenever it is dropped, kage says so on stderr, so the choice is never silent.
+- **Transient asset failures retry.** A download that hits a 403/429, a 5xx, or a network blip is retried with a short backoff, which recovers files that bot-protection rejects on the first request of a burst. Permanent failures like a 404 are not retried.
+- **Clearer crawl errors.** Each failure now logs a classified reason (`HTTP 403 Forbidden`, `timed out`, ...), the URL, and the page that referenced it, and the end-of-run summary lists what went wrong instead of printing only a count.
+- **The container image runs.** Chrome aborted in the image with `chrome_crashpad_handler: --database is required`, so the crash reporter is now disabled inside a container, and the `kage` user has a writable home (the mounted `/out` volume) so output, resume state, and Chrome's profile no longer fail with a permission error ([#7](https://github.com/tamnd/kage/issues/7)).
+
+## v0.1.1
+
+Packing, so a clone can travel as one file instead of a folder.
+
+- **`kage pack <mirror-dir>`** collapses a mirror into a single distributable file. `--format zim` (the default) writes an open ZIM archive, the same format [Kiwix](https://kiwix.org) uses, so the file opens in any ZIM reader and not just kage. `--format binary` appends that archive to a copy of kage to make a self-contained executable that serves the site offline when run. Packing is deterministic, so the same mirror produces a byte-identical file.
+- **`kage open <file.zim>`** serves a packed ZIM back over a local HTTP server, the read side of `kage pack --format zim`.
+- **An optional native-window viewer.** Built with `-tags webview`, `kage open` and a packed binary show the site in a real window backed by the operating system's WebView instead of a browser tab. The default build stays pure Go and opens the browser, so the release pipeline is unchanged.
+- **A pure-Go `zim` package** that reads and writes the ZIM format: a fixed header, MIME and pointer lists, zstd or stored clusters, redirects, and a trailing MD5.
 
 ## v0.1.0
 
-The first release. kage clones a live website into a self-contained folder you
-can browse offline, with every script stripped out.
+The first release. kage clones a live website into a self-contained folder you can browse offline, with every script stripped out.
 
-- **`kage clone <url>`** renders each page in headless Chrome, strips all
-  JavaScript, and localises CSS, images, and fonts to relative paths.
+- **`kage clone <url>`** renders each page in headless Chrome, strips all JavaScript, and localises CSS, images, and fonts to relative paths.
 - **`kage serve [dir]`** previews a cloned folder over a local file server.
-- **Idempotent and resumable.** Each page is keyed by the file it writes, so a
-  page reached over http and https, or as `/index.html` versus `/`, is fetched
-  once. Re-running resumes; `--refresh` re-renders in place; `--force` starts
-  clean.
-- **Polite by default.** Honours `robots.txt`, seeds from `sitemap.xml`, scopes
-  to the seed host, and runs three parallel worker tiers.
-- **Packaged everywhere.** Archives, `.deb`/`.rpm`/`.apk`, a multi-arch GHCR
-  image with Chromium bundled, checksums, SBOMs, and a cosign signature.
+- **Idempotent and resumable.** Each page is keyed by the file it writes, so a page reached over http and https, or as `/index.html` versus `/`, is fetched once. Re-running resumes; `--refresh` re-renders in place; `--force` starts clean.
+- **Polite by default.** Honours `robots.txt`, seeds from `sitemap.xml`, scopes to the seed host, and runs three parallel worker tiers.
+- **Packaged everywhere.** Archives, `.deb`/`.rpm`/`.apk`, a multi-arch GHCR image with Chromium bundled, checksums, SBOMs, and a cosign signature.

docs/demo/kage.tape

@@ -0,0 +1,34 @@
+# Demo tape for kage. Rendered with ascii-gif (github.com/tamnd/ascii-gif),
+# which supplies the window chrome and theme; this file is just the action.
+#
+#   ascii-gif render docs/demo/kage.tape -o docs/static/demo.gif
+#
+# kage must be on PATH inside the recording shell, and Chrome available.
+
+Hide
+Type "export PS1='$ ' PATH=/tmp/kagebin:$PATH && cd $(mktemp -d) && clear"
+Enter
+Show
+
+Sleep 600ms
+Type "kage clone paulgraham.com --max-pages 3 --out ."
+Sleep 700ms
+Enter
+Sleep 9s
+
+Type "kage pack ./paulgraham.com"
+Sleep 700ms
+Enter
+Sleep 2.5s
+
+Type "kage pack ./paulgraham.com --format binary -o paulgraham"
+Sleep 700ms
+Enter
+Sleep 3s
+
+Type "kage open paulgraham.com.zim --open=false"
+Sleep 700ms
+Enter
+Sleep 3s
+Ctrl+C
+Sleep 1.2s

go.mod

@@ -7,11 +7,17 @@ require (
 	github.com/charmbracelet/fang v1.0.0
 	github.com/go-rod/rod v0.116.2
 	github.com/go-rod/stealth v0.4.9
+	github.com/google/uuid v1.6.0
+	github.com/klauspost/compress v1.18.6
+	github.com/parquet-go/parquet-go v0.30.1
 	github.com/spf13/cobra v1.10.2
+	github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6
+	golang.org/x/image v0.42.0
 	golang.org/x/net v0.56.0
 )
 
 require (
+	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/charmbracelet/colorprofile v0.3.3 // indirect
 	github.com/charmbracelet/ultraviolet v0.0.0-20251106190538-99ea45596692 // indirect
 	github.com/charmbracelet/x/ansi v0.11.0 // indirect
@@ -30,8 +36,12 @@ require (
 	github.com/muesli/mango-cobra v1.2.0 // indirect
 	github.com/muesli/mango-pflag v0.1.0 // indirect
 	github.com/muesli/roff v0.1.0 // indirect
+	github.com/parquet-go/bitpack v1.0.0 // indirect
+	github.com/parquet-go/jsonlite v1.0.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
+	github.com/twpayne/go-geom v1.6.1 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/ysmood/fetchup v0.2.3 // indirect
 	github.com/ysmood/goob v0.4.0 // indirect
@@ -41,4 +51,5 @@ require (
 	golang.org/x/sync v0.21.0 // indirect
 	golang.org/x/sys v0.46.0 // indirect
 	golang.org/x/text v0.38.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 )

go.sum

@@ -1,5 +1,13 @@
 charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410 h1:D9PbaszZYpB4nj+d6HTWr1onlmlyuGVNfL9gAi8iB3k=
 charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410/go.mod h1:1qZyvvVCenJO2M1ac2mX0yyiIZJoZmDM4DG4s0udJkU=
+github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
+github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
+github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY=
+github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
 github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY=
 github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E=
 github.com/charmbracelet/colorprofile v0.3.3 h1:DjJzJtLP6/NZ8p7Cgjno0CKGr7wwRJGxWUwh2IyhfAI=
@@ -34,8 +42,16 @@ github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
 github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
 github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4=
 github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
+github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
 github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
@@ -50,6 +66,14 @@ github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe
 github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0=
 github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8=
 github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig=
+github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
+github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
+github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
+github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
+github.com/parquet-go/parquet-go v0.30.1 h1:Oy6ganNrAdFiVwy7wNmWagfPTWA2X9Z3tVHBc7JtuX8=
+github.com/parquet-go/parquet-go v0.30.1/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
@@ -61,8 +85,14 @@ github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4=
+github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
+github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6 h1:VQpB2SpK88C6B5lPHTuSZKb2Qee1QWwiFlC5CKY4AW0=
+github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6/go.mod h1:yE65LFCeWf4kyWD5re+h4XNvOHJEXOCOuJZ4v8l5sgk=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
 github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
 github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
@@ -83,6 +113,8 @@ github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/image v0.42.0 h1:1gSs6ehNWXLbkHBIPcWztk3D/6aIA/8hauiAYtlodVY=
+golang.org/x/image v0.42.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q=
 golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
 golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
 golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
@@ -91,6 +123,10 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
 golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
 golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pack/appbundle.go

@@ -0,0 +1,161 @@
+package pack
+
+import (
+	"encoding/xml"
+	"fmt"
+	"image"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// AppOptions controls how a macOS .app bundle is assembled around a packed
+// viewer.
+type AppOptions struct {
+	Out        string      // path to the .app directory
+	Base       string      // base kage binary (must be a macOS build); default os.Executable()
+	Name       string      // display name shown in Finder and the Dock
+	ExecName   string      // file name of the executable inside Contents/MacOS
+	Identifier string      // CFBundleIdentifier, e.g. com.kage.paulgraham
+	Version    string      // CFBundleShortVersionString; default 1.0
+	Icon       image.Image // optional; written as Resources/icon.icns
+}
+
+// BuildApp writes a double-clickable macOS application bundle that serves the
+// packed site. Finder runs Contents/MacOS/<ExecName> with no terminal attached,
+// which is the whole point: the bare appended binary opens a Terminal window
+// when double-clicked, a .app does not. The executable is the same
+// base++zim++trailer image BuildBinary produces, so Embedded finds the archive
+// at runtime exactly as it does for a plain viewer.
+//
+// It returns the bundle path and the size of the executable inside it (the part
+// that dominates, since the plist and icon are tiny).
+func BuildApp(zimBytes []byte, opts AppOptions) (string, int64, error) {
+	if opts.Out == "" {
+		return "", 0, fmt.Errorf("pack: BuildApp requires an output path")
+	}
+	if filepath.Ext(opts.Out) != ".app" {
+		return "", 0, fmt.Errorf("pack: app bundle path must end in .app, got %q", opts.Out)
+	}
+
+	base := opts.Base
+	if base == "" {
+		exe, err := os.Executable()
+		if err != nil {
+			return "", 0, fmt.Errorf("pack: locate base binary: %w", err)
+		}
+		base = exe
+	}
+	baseBytes, err := os.ReadFile(base)
+	if err != nil {
+		return "", 0, fmt.Errorf("pack: read base binary %q: %w", base, err)
+	}
+
+	execName := opts.ExecName
+	if execName == "" {
+		execName = "kage"
+	}
+	name := opts.Name
+	if name == "" {
+		name = execName
+	}
+	version := opts.Version
+	if version == "" {
+		version = "1.0"
+	}
+
+	// Start from a clean bundle so a rebuild never leaves a stale icon or
+	// executable behind. The path is known to end in .app from the guard above.
+	if err := os.RemoveAll(opts.Out); err != nil {
+		return "", 0, err
+	}
+	contents := filepath.Join(opts.Out, "Contents")
+	macOS := filepath.Join(contents, "MacOS")
+	resources := filepath.Join(contents, "Resources")
+	if err := os.MkdirAll(macOS, 0o755); err != nil {
+		return "", 0, err
+	}
+
+	hasIcon := opts.Icon != nil
+	if hasIcon {
+		if err := os.MkdirAll(resources, 0o755); err != nil {
+			return "", 0, err
+		}
+		icns, err := EncodeICNS(opts.Icon)
+		if err != nil {
+			return "", 0, err
+		}
+		if err := os.WriteFile(filepath.Join(resources, "icon.icns"), icns, 0o644); err != nil {
+			return "", 0, err
+		}
+	}
+
+	plist := infoPlist(infoPlistData{
+		Name:       name,
+		ExecName:   execName,
+		Identifier: opts.Identifier,
+		Version:    version,
+		HasIcon:    hasIcon,
+	})
+	if err := os.WriteFile(filepath.Join(contents, "Info.plist"), []byte(plist), 0o644); err != nil {
+		return "", 0, err
+	}
+	// PkgInfo is the eight-byte legacy type/creator stamp; APPL???? is the
+	// generic application value every modern bundle uses.
+	if err := os.WriteFile(filepath.Join(contents, "PkgInfo"), []byte("APPL????"), 0o644); err != nil {
+		return "", 0, err
+	}
+
+	exe := assemble(baseBytes, zimBytes)
+	if err := os.WriteFile(filepath.Join(macOS, execName), exe, 0o755); err != nil {
+		return "", 0, err
+	}
+	return opts.Out, int64(len(exe)), nil
+}
+
+type infoPlistData struct {
+	Name       string
+	ExecName   string
+	Identifier string
+	Version    string
+	HasIcon    bool
+}
+
+// infoPlist renders the bundle's Info.plist. Values are XML-escaped so a site
+// title with an ampersand or angle bracket cannot corrupt the file.
+func infoPlist(d infoPlistData) string {
+	pairs := [][2]string{
+		{"CFBundleName", d.Name},
+		{"CFBundleDisplayName", d.Name},
+		{"CFBundleExecutable", d.ExecName},
+		{"CFBundleIdentifier", d.Identifier},
+		{"CFBundleInfoDictionaryVersion", "6.0"},
+		{"CFBundlePackageType", "APPL"},
+		{"CFBundleShortVersionString", d.Version},
+		{"CFBundleVersion", d.Version},
+		{"LSMinimumSystemVersion", "10.13"},
+	}
+	if d.HasIcon {
+		pairs = append(pairs, [2]string{"CFBundleIconFile", "icon"})
+	}
+
+	var b strings.Builder
+	b.WriteString(xml.Header)
+	b.WriteString(`<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">` + "\n")
+	b.WriteString(`<plist version="1.0">` + "\n<dict>\n")
+	for _, kv := range pairs {
+		b.WriteString("\t<key>" + esc(kv[0]) + "</key>\n")
+		b.WriteString("\t<string>" + esc(kv[1]) + "</string>\n")
+	}
+	// NSHighResolutionCapable is a boolean, not a string, so the icon and text
+	// render sharp on Retina displays.
+	b.WriteString("\t<key>NSHighResolutionCapable</key>\n\t<true/>\n")
+	b.WriteString("</dict>\n</plist>\n")
+	return b.String()
+}
+
+func esc(s string) string {
+	var b strings.Builder
+	_ = xml.EscapeText(&b, []byte(s))
+	return b.String()
+}

pack/appbundle_test.go

@@ -0,0 +1,131 @@
+package pack
+
+import (
+	"encoding/binary"
+	"image/color"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func TestBuildApp(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "kage-darwin")
+	baseBytes := []byte("\xcf\xfa\xed\xfeMACHO-BASE-BYTES")
+	if err := os.WriteFile(base, baseBytes, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	zim := []byte("ZIM-ARCHIVE-PAYLOAD")
+	out := filepath.Join(dir, "Paulgraham.app")
+
+	path, size, err := BuildApp(zim, AppOptions{
+		Out:        out,
+		Base:       base,
+		Name:       "Paul Graham",
+		ExecName:   "paulgraham",
+		Identifier: "com.kage.paulgraham",
+		Version:    "1.0",
+		Icon:       solid(48, 48, color.RGBA{A: 0xff}),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if path != out {
+		t.Errorf("path = %q, want %q", path, out)
+	}
+
+	// The bundle skeleton.
+	exe := filepath.Join(out, "Contents", "MacOS", "paulgraham")
+	for _, f := range []string{
+		filepath.Join(out, "Contents", "Info.plist"),
+		filepath.Join(out, "Contents", "PkgInfo"),
+		filepath.Join(out, "Contents", "Resources", "icon.icns"),
+		exe,
+	} {
+		if _, err := os.Stat(f); err != nil {
+			t.Errorf("missing bundle file %s: %v", f, err)
+		}
+	}
+
+	// The executable is base++zim++trailer, and size reports its length.
+	exeBytes, err := os.ReadFile(exe)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if int64(len(exeBytes)) != size {
+		t.Errorf("size = %d, want %d", size, len(exeBytes))
+	}
+	if !strings.HasPrefix(string(exeBytes), string(baseBytes)) {
+		t.Error("executable does not start with the base bytes")
+	}
+	tr := exeBytes[len(exeBytes)-trailerLen:]
+	if string(tr[:8]) != trailerMagic || string(tr[trailerLen-8:]) != trailerMagic {
+		t.Error("trailer magic missing from the executable")
+	}
+	if got := binary.LittleEndian.Uint64(tr[8:16]); got != uint64(len(zim)) {
+		t.Errorf("trailer records zim length %d, want %d", got, len(zim))
+	}
+
+	// On a unix host the executable bit must be set so Finder can launch it.
+	if runtime.GOOS != "windows" {
+		info, err := os.Stat(exe)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if info.Mode().Perm()&0o111 == 0 {
+			t.Errorf("executable mode = %v, want the execute bit set", info.Mode().Perm())
+		}
+	}
+
+	// Info.plist carries the identity and points at the icon.
+	plist, err := os.ReadFile(filepath.Join(out, "Contents", "Info.plist"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{
+		"<string>paulgraham</string>",          // CFBundleExecutable
+		"<string>Paul Graham</string>",         // CFBundleName / DisplayName
+		"<string>com.kage.paulgraham</string>", // CFBundleIdentifier
+		"<key>CFBundleIconFile</key>",
+		"<string>icon</string>",
+		"NSHighResolutionCapable",
+	} {
+		if !strings.Contains(string(plist), want) {
+			t.Errorf("Info.plist missing %q", want)
+		}
+	}
+}
+
+func TestBuildAppNoIcon(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "kage")
+	if err := os.WriteFile(base, []byte("\xcf\xfa\xed\xfeBASE"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	out := filepath.Join(dir, "Site.app")
+	if _, _, err := BuildApp([]byte("ZIM"), AppOptions{Out: out, Base: base, ExecName: "site"}); err != nil {
+		t.Fatal(err)
+	}
+	// No icon means no Resources dir and no icon key in the plist.
+	if _, err := os.Stat(filepath.Join(out, "Contents", "Resources")); !os.IsNotExist(err) {
+		t.Errorf("Resources dir should be absent without an icon, got err=%v", err)
+	}
+	plist, err := os.ReadFile(filepath.Join(out, "Contents", "Info.plist"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(plist), "CFBundleIconFile") {
+		t.Error("plist names an icon file but none was provided")
+	}
+}
+
+func TestBuildAppRejectsBadOut(t *testing.T) {
+	if _, _, err := BuildApp([]byte("ZIM"), AppOptions{Out: "noapp", Base: "x"}); err == nil {
+		t.Error("BuildApp should reject an output path without a .app extension")
+	}
+	if _, _, err := BuildApp([]byte("ZIM"), AppOptions{Out: ""}); err == nil {
+		t.Error("BuildApp should reject an empty output path")
+	}
+}

pack/appdir.go

@@ -0,0 +1,156 @@
+package pack
+
+import (
+	"bytes"
+	"fmt"
+	"image"
+	"image/png"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// LinuxAppOptions controls how a Linux application directory is assembled around
+// a packed viewer.
+type LinuxAppOptions struct {
+	Out      string      // path to the .AppDir directory
+	Base     string      // base kage binary (must be a Linux build); default os.Executable()
+	Name     string      // display name shown in menus
+	ExecName string      // base name for the .desktop and icon files
+	Comment  string      // optional one-line description for the launcher
+	Version  string      // version string recorded in the .desktop
+	Icon     image.Image // optional; written as the launcher icon
+}
+
+// BuildAppDir writes an AppImage-style application directory. The layout follows
+// the AppDir convention so `appimagetool` can fold it into a single
+// double-clickable .AppImage, but it is useful on its own: AppRun is the packed
+// viewer, the .desktop file launches it with Terminal=false (no console), and
+// the icon gives it a face in the file manager and menus.
+//
+// It returns the AppDir path, the size of the executable inside it, and whether
+// an icon was written (the caller needs that to decide if an .AppImage can be
+// built, since AppImage requires one).
+func BuildAppDir(zimBytes []byte, opts LinuxAppOptions) (path string, size int64, hasIcon bool, err error) {
+	if opts.Out == "" {
+		return "", 0, false, fmt.Errorf("pack: BuildAppDir requires an output path")
+	}
+	if !strings.HasSuffix(opts.Out, ".AppDir") {
+		return "", 0, false, fmt.Errorf("pack: app dir path must end in .AppDir, got %q", opts.Out)
+	}
+
+	base := opts.Base
+	if base == "" {
+		exe, e := os.Executable()
+		if e != nil {
+			return "", 0, false, fmt.Errorf("pack: locate base binary: %w", e)
+		}
+		base = exe
+	}
+	baseBytes, err := os.ReadFile(base)
+	if err != nil {
+		return "", 0, false, fmt.Errorf("pack: read base binary %q: %w", base, err)
+	}
+
+	execName := opts.ExecName
+	if execName == "" {
+		execName = "kage"
+	}
+	name := opts.Name
+	if name == "" {
+		name = execName
+	}
+
+	if err := os.RemoveAll(opts.Out); err != nil {
+		return "", 0, false, err
+	}
+	if err := os.MkdirAll(opts.Out, 0o755); err != nil {
+		return "", 0, false, err
+	}
+
+	// AppRun is the AppImage entrypoint; pointing it straight at the packed
+	// binary keeps the directory to a single executable with no wrapper script.
+	exe := assemble(baseBytes, zimBytes)
+	if err := os.WriteFile(filepath.Join(opts.Out, "AppRun"), exe, 0o755); err != nil {
+		return "", 0, false, err
+	}
+
+	hasIcon = opts.Icon != nil
+	if hasIcon {
+		pngBytes, err := encodeIconPNG(opts.Icon, 256)
+		if err != nil {
+			return "", 0, false, err
+		}
+		// The icon lives at the root under the name the .desktop references, and
+		// .DirIcon is the AppImage convention for the directory's own icon.
+		if err := os.WriteFile(filepath.Join(opts.Out, execName+".png"), pngBytes, 0o644); err != nil {
+			return "", 0, false, err
+		}
+		if err := os.WriteFile(filepath.Join(opts.Out, ".DirIcon"), pngBytes, 0o644); err != nil {
+			return "", 0, false, err
+		}
+	}
+
+	desktop := desktopEntry(desktopData{
+		Name:     name,
+		ExecName: execName,
+		Comment:  opts.Comment,
+		Version:  opts.Version,
+		HasIcon:  hasIcon,
+	})
+	if err := os.WriteFile(filepath.Join(opts.Out, execName+".desktop"), []byte(desktop), 0o644); err != nil {
+		return "", 0, false, err
+	}
+
+	return opts.Out, int64(len(exe)), hasIcon, nil
+}
+
+type desktopData struct {
+	Name     string
+	ExecName string
+	Comment  string
+	Version  string
+	HasIcon  bool
+}
+
+// desktopEntry renders a freedesktop .desktop launcher. Terminal=false is the
+// line that keeps a console from opening, the Linux echo of the macOS .app.
+func desktopEntry(d desktopData) string {
+	var b strings.Builder
+	b.WriteString("[Desktop Entry]\n")
+	b.WriteString("Type=Application\n")
+	b.WriteString("Name=" + desktopValue(d.Name) + "\n")
+	if d.Comment != "" {
+		b.WriteString("Comment=" + desktopValue(d.Comment) + "\n")
+	}
+	// AppRun is on PATH inside a running AppImage, so Exec names it directly.
+	b.WriteString("Exec=AppRun\n")
+	if d.HasIcon {
+		b.WriteString("Icon=" + d.ExecName + "\n")
+	}
+	b.WriteString("Categories=Network;Utility;\n")
+	b.WriteString("Terminal=false\n")
+	if d.Version != "" {
+		b.WriteString("X-AppImage-Version=" + desktopValue(d.Version) + "\n")
+	}
+	return b.String()
+}
+
+// desktopValue strips the characters that would break a .desktop key=value line
+// (newlines and the leading-space/escape pitfalls), which is enough for a name
+// or comment drawn from a page title.
+func desktopValue(s string) string {
+	s = strings.ReplaceAll(s, "\n", " ")
+	s = strings.ReplaceAll(s, "\r", " ")
+	return strings.TrimSpace(s)
+}
+
+// encodeIconPNG scales img to a px-by-px square and encodes it as PNG, the icon
+// format the freedesktop world expects.
+func encodeIconPNG(img image.Image, px int) ([]byte, error) {
+	var buf bytes.Buffer
+	if err := png.Encode(&buf, scaleSquare(img, px)); err != nil {
+		return nil, fmt.Errorf("pack: encode icon png: %w", err)
+	}
+	return buf.Bytes(), nil
+}

pack/appdir_test.go

@@ -0,0 +1,125 @@
+package pack
+
+import (
+	"image/color"
+	"image/png"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestBuildAppDir(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "kage-linux")
+	baseBytes := []byte("\x7fELF-LINUX-BASE")
+	if err := os.WriteFile(base, baseBytes, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	out := filepath.Join(dir, "paulgraham.AppDir")
+
+	path, size, hasIcon, err := BuildAppDir([]byte("ZIM-PAYLOAD"), LinuxAppOptions{
+		Out:      out,
+		Base:     base,
+		Name:     "Paul Graham",
+		ExecName: "paulgraham",
+		Comment:  "Offline mirror",
+		Version:  "1.0",
+		Icon:     solid(64, 64, color.RGBA{B: 0xff, A: 0xff}),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if path != out || !hasIcon {
+		t.Fatalf("path=%q hasIcon=%v", path, hasIcon)
+	}
+
+	apprun := filepath.Join(out, "AppRun")
+	for _, f := range []string{
+		apprun,
+		filepath.Join(out, "paulgraham.desktop"),
+		filepath.Join(out, "paulgraham.png"),
+		filepath.Join(out, ".DirIcon"),
+	} {
+		if _, err := os.Stat(f); err != nil {
+			t.Errorf("missing %s: %v", f, err)
+		}
+	}
+
+	exe, err := os.ReadFile(apprun)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if int64(len(exe)) != size {
+		t.Errorf("size = %d, want %d", size, len(exe))
+	}
+	if !strings.HasPrefix(string(exe), string(baseBytes)) {
+		t.Error("AppRun does not start with the base bytes")
+	}
+	tr := exe[len(exe)-trailerLen:]
+	if string(tr[:8]) != trailerMagic {
+		t.Error("AppRun missing the trailer")
+	}
+	if info, _ := os.Stat(apprun); info.Mode().Perm()&0o111 == 0 {
+		t.Error("AppRun is not executable")
+	}
+
+	// The icon is a real 256px PNG.
+	f, err := os.Open(filepath.Join(out, "paulgraham.png"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = f.Close() }()
+	img, err := png.Decode(f)
+	if err != nil {
+		t.Fatalf("icon is not a PNG: %v", err)
+	}
+	if img.Bounds().Dx() != 256 {
+		t.Errorf("icon width = %d, want 256", img.Bounds().Dx())
+	}
+
+	desktop, err := os.ReadFile(filepath.Join(out, "paulgraham.desktop"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{
+		"Name=Paul Graham",
+		"Exec=AppRun",
+		"Icon=paulgraham",
+		"Terminal=false",
+		"Comment=Offline mirror",
+	} {
+		if !strings.Contains(string(desktop), want) {
+			t.Errorf(".desktop missing %q", want)
+		}
+	}
+}
+
+func TestBuildAppDirNoIcon(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "kage")
+	if err := os.WriteFile(base, []byte("\x7fELF"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	out := filepath.Join(dir, "site.AppDir")
+	_, _, hasIcon, err := BuildAppDir([]byte("ZIM"), LinuxAppOptions{Out: out, Base: base, ExecName: "site"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hasIcon {
+		t.Error("hasIcon should be false without an icon")
+	}
+	if _, err := os.Stat(filepath.Join(out, "site.png")); !os.IsNotExist(err) {
+		t.Errorf("icon should be absent, got err=%v", err)
+	}
+	desktop, _ := os.ReadFile(filepath.Join(out, "site.desktop"))
+	if strings.Contains(string(desktop), "Icon=") {
+		t.Error(".desktop names an icon but none was written")
+	}
+}
+
+func TestBuildAppDirRejectsBadOut(t *testing.T) {
+	if _, _, _, err := BuildAppDir([]byte("Z"), LinuxAppOptions{Out: "site", Base: "x"}); err == nil {
+		t.Error("BuildAppDir should reject a path without .AppDir")
+	}
+}

pack/binary.go

@@ -0,0 +1,74 @@
+package pack
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"os"
+)
+
+const (
+	// trailerMagic brackets the appended-archive trailer at both ends, so a stray
+	// copy of it inside the base binary cannot be mistaken for a real trailer.
+	trailerMagic = "KAGEPCK1"
+	// trailerLen is magic + uint64 archive length + magic again.
+	trailerLen = len(trailerMagic) + 8 + len(trailerMagic)
+)
+
+// BinaryOptions controls how a self-contained viewer is assembled.
+type BinaryOptions struct {
+	Out  string // output path
+	Base string // base kage binary; default os.Executable()
+}
+
+// BuildBinary writes baseExe ++ zimBytes ++ trailer to opts.Out and marks it
+// executable. The base must be a kage binary, since the viewer behaviour lives
+// in kage's own startup hook (see Embedded); appending a ZIM to an arbitrary
+// executable would only produce a broken file. It returns the output path and
+// the total byte size.
+func BuildBinary(zimBytes []byte, opts BinaryOptions) (string, int64, error) {
+	base := opts.Base
+	if base == "" {
+		exe, err := os.Executable()
+		if err != nil {
+			return "", 0, fmt.Errorf("pack: locate base binary: %w", err)
+		}
+		base = exe
+	}
+	if opts.Out == "" {
+		return "", 0, fmt.Errorf("pack: BuildBinary requires an output path")
+	}
+
+	baseBytes, err := os.ReadFile(base)
+	if err != nil {
+		return "", 0, fmt.Errorf("pack: read base binary %q: %w", base, err)
+	}
+
+	payload := assemble(baseBytes, zimBytes)
+	if err := os.WriteFile(opts.Out, payload, 0o755); err != nil {
+		return opts.Out, 0, err
+	}
+	// WriteFile honours the mode only when it creates the file; chmod makes an
+	// overwrite executable too.
+	if err := os.Chmod(opts.Out, 0o755); err != nil {
+		return opts.Out, 0, err
+	}
+	return opts.Out, int64(len(payload)), nil
+}
+
+// assemble builds the self-contained viewer image: the base executable, then the
+// ZIM archive, then the KAGEPCK1 trailer that records the archive length. ELF,
+// PE, and Mach-O loaders all ignore trailing bytes, so the result still runs on
+// its target OS while Embedded finds the archive at the tail.
+func assemble(baseBytes, zimBytes []byte) []byte {
+	var tr bytes.Buffer
+	tr.WriteString(trailerMagic)
+	_ = binary.Write(&tr, binary.LittleEndian, uint64(len(zimBytes)))
+	tr.WriteString(trailerMagic)
+
+	out := make([]byte, 0, len(baseBytes)+len(zimBytes)+tr.Len())
+	out = append(out, baseBytes...)
+	out = append(out, zimBytes...)
+	out = append(out, tr.Bytes()...)
+	return out
+}

pack/cache.go

@@ -0,0 +1,134 @@
+package pack
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"io"
+	"os"
+	"sort"
+
+	"github.com/tamnd/kage/zim"
+)
+
+// cacheMagic tags the sidecar so a stale or foreign file is recognised and
+// ignored rather than misread. The trailing digit is the format version.
+var cacheMagic = [8]byte{'k', 'a', 'g', 'e', 'z', 'c', 'h', '1'}
+
+// clusterCache is a content-addressed store of compressed ZIM clusters, kept in
+// a sidecar next to the archive. Compressing clusters with zstd is the dominant
+// cost of packing a large mirror, so a re-pack reuses the compression of every
+// cluster whose uncompressed bytes are unchanged and only compresses the rest.
+//
+// A cache hit returns exactly what a fresh compression would have produced, so
+// the archive stays deterministic and valid; the cache only saves CPU. Clusters
+// are keyed by the SHA-256 of their uncompressed data section, which the zim
+// writer assembles before compression.
+type clusterCache struct {
+	prev map[[32]byte][]byte // clusters loaded from the previous pack
+	used map[[32]byte][]byte // clusters touched this pack, written back on save
+
+	reused     int // clusters served from the cache this pack
+	compressed int // clusters compressed fresh this pack
+}
+
+func newClusterCache() *clusterCache {
+	return &clusterCache{prev: map[[32]byte][]byte{}, used: map[[32]byte][]byte{}}
+}
+
+// loadClusterCache reads the sidecar at path. A missing, unreadable, or
+// truncated cache is not an error: packing starts cold and rebuilds it.
+func loadClusterCache(path string) *clusterCache {
+	c := newClusterCache()
+	f, err := os.Open(path)
+	if err != nil {
+		return c
+	}
+	defer func() { _ = f.Close() }()
+
+	r := bufio.NewReader(f)
+	var magic [8]byte
+	if _, err := io.ReadFull(r, magic[:]); err != nil || magic != cacheMagic {
+		return c
+	}
+	var count uint32
+	if err := binary.Read(r, binary.LittleEndian, &count); err != nil {
+		return c
+	}
+	for i := uint32(0); i < count; i++ {
+		var h [32]byte
+		if _, err := io.ReadFull(r, h[:]); err != nil {
+			return newClusterCache() // truncated: drop the partial load
+		}
+		var n uint32
+		if err := binary.Read(r, binary.LittleEndian, &n); err != nil {
+			return newClusterCache()
+		}
+		b := make([]byte, n)
+		if _, err := io.ReadFull(r, b); err != nil {
+			return newClusterCache()
+		}
+		c.prev[h] = b
+	}
+	return c
+}
+
+// Compress returns the stored bytes for an uncompressed cluster: the cached
+// compression when the cluster's bytes are unchanged, a fresh zstd compression
+// on a miss. It is the function handed to zim.Writer.SetCompress.
+func (c *clusterCache) Compress(data []byte) []byte {
+	h := sha256.Sum256(data)
+	if b, ok := c.used[h]; ok {
+		return b
+	}
+	if b, ok := c.prev[h]; ok {
+		c.used[h] = b
+		c.reused++
+		return b
+	}
+	b := zim.Compress(data)
+	c.used[h] = b
+	c.compressed++
+	return b
+}
+
+// save writes the clusters touched this pack to path, replacing the previous
+// sidecar. Only touched clusters are written, so clusters that left the mirror
+// drop out and the cache cannot grow without bound. Entries are sorted by hash
+// so the sidecar itself is reproducible. The write goes through a temp file and
+// a rename so a crash mid-write cannot corrupt an existing cache.
+func (c *clusterCache) save(path string) error {
+	hashes := make([][32]byte, 0, len(c.used))
+	for h := range c.used {
+		hashes = append(hashes, h)
+	}
+	sort.Slice(hashes, func(i, j int) bool {
+		return bytes.Compare(hashes[i][:], hashes[j][:]) < 0
+	})
+
+	tmp := path + ".tmp"
+	f, err := os.Create(tmp)
+	if err != nil {
+		return err
+	}
+	w := bufio.NewWriter(f)
+	_, _ = w.Write(cacheMagic[:])
+	_ = binary.Write(w, binary.LittleEndian, uint32(len(hashes)))
+	for _, h := range hashes {
+		b := c.used[h]
+		_, _ = w.Write(h[:])
+		_ = binary.Write(w, binary.LittleEndian, uint32(len(b)))
+		_, _ = w.Write(b)
+	}
+	if err := w.Flush(); err != nil {
+		_ = f.Close()
+		_ = os.Remove(tmp)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		_ = os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, path)
+}

pack/cache_test.go

@@ -0,0 +1,122 @@
+package pack
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/tamnd/kage/zim"
+)
+
+// TestIncrementalPackReusesCompression checks the core promise of the cache: a
+// second pack of an unchanged mirror compresses nothing, reuses every cluster,
+// and produces a byte-identical archive.
+func TestIncrementalPackReusesCompression(t *testing.T) {
+	host := writeMirror(t)
+	dir := t.TempDir()
+	out := filepath.Join(dir, "example.zim")
+	cache := out + ".kagecache"
+
+	var first PackStats
+	a, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &first})
+	if err != nil {
+		t.Fatalf("first pack: %v", err)
+	}
+	if first.ClustersCompressed == 0 {
+		t.Fatal("first pack reported no clusters compressed")
+	}
+	if first.ClustersReused != 0 {
+		t.Errorf("first pack reused %d clusters, want 0", first.ClustersReused)
+	}
+	if _, err := os.Stat(cache); err != nil {
+		t.Fatalf("cache sidecar not written: %v", err)
+	}
+	bytesA, err := os.ReadFile(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var second PackStats
+	b, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &second})
+	if err != nil {
+		t.Fatalf("second pack: %v", err)
+	}
+	if second.ClustersCompressed != 0 {
+		t.Errorf("second pack compressed %d clusters, want 0 (all reused)", second.ClustersCompressed)
+	}
+	if second.ClustersReused != first.ClustersCompressed {
+		t.Errorf("second pack reused %d clusters, want %d", second.ClustersReused, first.ClustersCompressed)
+	}
+	bytesB, err := os.ReadFile(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(bytesA, bytesB) {
+		t.Error("a cached re-pack produced a different archive than the cold pack")
+	}
+}
+
+// TestIncrementalPackRecompressesChange checks that editing one page forces a
+// fresh compression of the cluster that holds it while the rest are reused, and
+// that the archive still opens and serves the new content.
+func TestIncrementalPackRecompressesChange(t *testing.T) {
+	host := writeMirror(t)
+	dir := t.TempDir()
+	out := filepath.Join(dir, "example.zim")
+	cache := out + ".kagecache"
+
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache}); err != nil {
+		t.Fatalf("first pack: %v", err)
+	}
+
+	// Change one page. Its text cluster must be recompressed.
+	edited := filepath.Join(host, "about", "index.html")
+	if err := os.WriteFile(edited, []byte("<!doctype html><title>About</title><h1>Changed</h1>"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	var st PackStats
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &st}); err != nil {
+		t.Fatalf("second pack: %v", err)
+	}
+	if st.ClustersCompressed == 0 {
+		t.Error("editing a page compressed nothing; expected the changed cluster to be recompressed")
+	}
+
+	r, err := zim.Open(out)
+	if err != nil {
+		t.Fatalf("reopen: %v", err)
+	}
+	defer func() { _ = r.Close() }()
+	about, err := r.Get(zim.NamespaceContent, "about/index.html")
+	if err != nil {
+		t.Fatalf("about page missing after re-pack: %v", err)
+	}
+	if !bytes.Contains(about.Data, []byte("Changed")) {
+		t.Errorf("re-packed page did not carry the edit: %q", about.Data)
+	}
+}
+
+// TestClusterCacheSurvivesCorruptSidecar checks that a damaged cache file is
+// treated as a cold start rather than failing the pack.
+func TestClusterCacheSurvivesCorruptSidecar(t *testing.T) {
+	host := writeMirror(t)
+	dir := t.TempDir()
+	out := filepath.Join(dir, "example.zim")
+	cache := out + ".kagecache"
+	if err := os.WriteFile(cache, []byte("not a real cache file"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	var st PackStats
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &st}); err != nil {
+		t.Fatalf("pack over corrupt cache: %v", err)
+	}
+	if st.ClustersReused != 0 {
+		t.Errorf("corrupt cache should reuse nothing, reused %d", st.ClustersReused)
+	}
+	if st.ClustersCompressed == 0 {
+		t.Error("pack over corrupt cache compressed nothing")
+	}
+}

pack/embed.go

@@ -0,0 +1,50 @@
+package pack
+
+import (
+	"encoding/binary"
+	"io"
+	"os"
+)
+
+// Embedded inspects the running executable for an appended ZIM archive. If the
+// KAGEPCK1 trailer is present, it returns a ReaderAt bounded to the archive, its
+// size, and ok=true; the file handle stays open for the life of the process so
+// the viewer can serve from it. A normal kage build has no trailer, so the cost
+// to every ordinary invocation is one Open plus a 24-byte ReadAt.
+func Embedded() (ra io.ReaderAt, size int64, ok bool) {
+	exe, err := os.Executable()
+	if err != nil {
+		return nil, 0, false
+	}
+	f, err := os.Open(exe)
+	if err != nil {
+		return nil, 0, false
+	}
+	info, err := f.Stat()
+	if err != nil {
+		_ = f.Close()
+		return nil, 0, false
+	}
+	total := info.Size()
+	if total < int64(trailerLen) {
+		_ = f.Close()
+		return nil, 0, false
+	}
+
+	tr := make([]byte, trailerLen)
+	if _, err := f.ReadAt(tr, total-int64(trailerLen)); err != nil {
+		_ = f.Close()
+		return nil, 0, false
+	}
+	if string(tr[:8]) != trailerMagic || string(tr[trailerLen-8:]) != trailerMagic {
+		_ = f.Close()
+		return nil, 0, false
+	}
+	zlen := int64(binary.LittleEndian.Uint64(tr[8:16]))
+	start := total - int64(trailerLen) - zlen
+	if zlen <= 0 || start < 0 {
+		_ = f.Close()
+		return nil, 0, false
+	}
+	return io.NewSectionReader(f, start, zlen), zlen, true
+}

pack/icns.go

@@ -0,0 +1,92 @@
+package pack
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"image"
+	"image/png"
+
+	xdraw "golang.org/x/image/draw"
+)
+
+// An .icns file is a tiny container: the magic "icns", a uint32 big-endian total
+// length, then a run of entries. Each entry is a four-byte OSType, a uint32
+// big-endian length covering the 8-byte entry header plus the payload, and the
+// payload itself. Since OS X 10.7 the payload may be a PNG, which lets us avoid
+// the old packed-RGBA formats entirely and just store one PNG per size.
+//
+// We emit the retina-era PNG OSTypes. macOS picks whichever size it needs for
+// the Dock, Finder, and Cmd-Tab, so covering 16 through 1024 keeps the icon
+// crisp everywhere without shipping a huge file.
+var icnsSizes = []struct {
+	osType string
+	px     int
+}{
+	{"icp4", 16},
+	{"icp5", 32},
+	{"icp6", 64},
+	{"ic07", 128},
+	{"ic08", 256},
+	{"ic09", 512},
+	{"ic10", 1024},
+}
+
+// EncodeICNS renders img into a macOS .icns at every standard size. The source
+// is scaled to each size with Catmull-Rom resampling, which keeps a small
+// favicon from turning to mush when it is enlarged for the Dock. It returns an
+// error only if img is empty or a PNG fails to encode.
+func EncodeICNS(img image.Image) ([]byte, error) {
+	if img == nil || img.Bounds().Empty() {
+		return nil, fmt.Errorf("pack: icns source image is empty")
+	}
+
+	var body bytes.Buffer
+	for _, s := range icnsSizes {
+		scaled := scaleSquare(img, s.px)
+		var pngBuf bytes.Buffer
+		if err := png.Encode(&pngBuf, scaled); err != nil {
+			return nil, fmt.Errorf("pack: encode %s icon: %w", s.osType, err)
+		}
+		body.WriteString(s.osType)
+		writeU32BE(&body, uint32(8+pngBuf.Len()))
+		body.Write(pngBuf.Bytes())
+	}
+
+	var out bytes.Buffer
+	out.WriteString("icns")
+	writeU32BE(&out, uint32(8+body.Len()))
+	out.Write(body.Bytes())
+	return out.Bytes(), nil
+}
+
+// scaleSquare returns img resampled to a px-by-px RGBA image. A non-square
+// source is fitted into the square and centred, so a wide or tall favicon is not
+// stretched.
+func scaleSquare(img image.Image, px int) image.Image {
+	dst := image.NewRGBA(image.Rect(0, 0, px, px))
+	src := img.Bounds()
+	sw, sh := src.Dx(), src.Dy()
+	// Scale to fit, preserving aspect ratio.
+	scale := float64(px) / float64(sw)
+	if float64(sh)*scale > float64(px) {
+		scale = float64(px) / float64(sh)
+	}
+	dw, dh := int(float64(sw)*scale), int(float64(sh)*scale)
+	if dw < 1 {
+		dw = 1
+	}
+	if dh < 1 {
+		dh = 1
+	}
+	off := image.Pt((px-dw)/2, (px-dh)/2)
+	rect := image.Rectangle{Min: off, Max: off.Add(image.Pt(dw, dh))}
+	xdraw.CatmullRom.Scale(dst, rect, img, src, xdraw.Over, nil)
+	return dst
+}
+
+func writeU32BE(w *bytes.Buffer, v uint32) {
+	var b [4]byte
+	binary.BigEndian.PutUint32(b[:], v)
+	w.Write(b[:])
+}

pack/icns_test.go

@@ -0,0 +1,73 @@
+package pack
+
+import (
+	"bytes"
+	"encoding/binary"
+	"image"
+	"image/color"
+	"image/png"
+	"testing"
+)
+
+// solid returns a w-by-h image filled with one colour, enough to exercise the
+// encoder without depending on any fixture file.
+func solid(w, h int, c color.Color) image.Image {
+	img := image.NewRGBA(image.Rect(0, 0, w, h))
+	for y := 0; y < h; y++ {
+		for x := 0; x < w; x++ {
+			img.Set(x, y, c)
+		}
+	}
+	return img
+}
+
+func TestEncodeICNS(t *testing.T) {
+	data, err := EncodeICNS(solid(48, 48, color.RGBA{R: 0x33, G: 0x66, B: 0x99, A: 0xff}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data[:4]) != "icns" {
+		t.Fatalf("magic = %q, want icns", data[:4])
+	}
+	if got := binary.BigEndian.Uint32(data[4:8]); int(got) != len(data) {
+		t.Fatalf("header length = %d, want %d", got, len(data))
+	}
+
+	// Walk the entries and confirm each declared OSType is present and its PNG
+	// decodes to the size it claims.
+	seen := map[string]int{}
+	off := 8
+	for off < len(data) {
+		osType := string(data[off : off+4])
+		size := int(binary.BigEndian.Uint32(data[off+4 : off+8]))
+		if size < 8 || off+size > len(data) {
+			t.Fatalf("entry %s has bogus length %d at offset %d", osType, size, off)
+		}
+		img, err := png.Decode(bytes.NewReader(data[off+8 : off+size]))
+		if err != nil {
+			t.Fatalf("entry %s payload is not a PNG: %v", osType, err)
+		}
+		seen[osType] = img.Bounds().Dx()
+		off += size
+	}
+
+	for _, s := range icnsSizes {
+		px, ok := seen[s.osType]
+		if !ok {
+			t.Errorf("missing OSType %s", s.osType)
+			continue
+		}
+		if px != s.px {
+			t.Errorf("OSType %s is %dpx, want %d", s.osType, px, s.px)
+		}
+	}
+}
+
+func TestEncodeICNSEmpty(t *testing.T) {
+	if _, err := EncodeICNS(nil); err == nil {
+		t.Fatal("EncodeICNS(nil) should error")
+	}
+	if _, err := EncodeICNS(image.NewRGBA(image.Rect(0, 0, 0, 0))); err == nil {
+		t.Fatal("EncodeICNS(empty) should error")
+	}
+}

pack/icon.go

@@ -0,0 +1,329 @@
+package pack
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"image"
+	"image/color"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	// Register the decoders image.Decode dispatches to. A favicon is almost
+	// always one of these; SVG is skipped explicitly, and a BMP-in-ICO is
+	// decoded by hand below since the stdlib has no BMP decoder.
+	_ "image/gif"
+	_ "image/jpeg"
+	"image/png"
+)
+
+// iconNames ranks the file names sites use for their icon, best first. A large
+// PNG (an apple-touch icon, typically 180px) makes a far better app icon than a
+// 16px favicon.ico, so we prefer those even though .ico is the classic name.
+var iconNames = []string{
+	"apple-touch-icon-precomposed.png",
+	"apple-touch-icon.png",
+	"icon.png",
+	"favicon.png",
+	"favicon.ico",
+}
+
+// FindIcon looks through a cloned mirror for the site's icon and decodes it. It
+// returns the image, the path it came from (for a friendly log line), and
+// ok=false when nothing usable is found, in which case the caller just builds a
+// bundle with the default icon. Discovery never fails the pack.
+func FindIcon(mirrorDir string) (image.Image, string, bool) {
+	for _, name := range iconNames {
+		for _, p := range globIcon(mirrorDir, name) {
+			if img, err := DecodeIcon(p); err == nil {
+				return img, p, true
+			}
+		}
+	}
+	return nil, "", false
+}
+
+// Favicon48 finds the mirror's icon and renders it to a 48x48 PNG, the form the
+// ZIM Illustrator_48x48@1 metadata takes and the icon Kiwix shows for the book.
+// It returns ok=false when the mirror has no usable icon, in which case the
+// archive simply ships without one rather than failing the pack.
+func Favicon48(mirrorDir string) ([]byte, bool) {
+	img, _, ok := FindIcon(mirrorDir)
+	if !ok {
+		return nil, false
+	}
+	var buf bytes.Buffer
+	if err := png.Encode(&buf, scaleSquare(img, 48)); err != nil {
+		return nil, false
+	}
+	return buf.Bytes(), true
+}
+
+// globIcon returns every file under dir whose base name equals name, nearest the
+// root first. Clones store assets under rewritten paths, so the icon may sit a
+// few directories deep rather than at the mirror root.
+func globIcon(dir, name string) []string {
+	var hits []string
+	_ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() {
+			return nil
+		}
+		if strings.EqualFold(d.Name(), name) {
+			hits = append(hits, p)
+		}
+		return nil
+	})
+	// Shallower paths (fewer separators) are likelier to be the real site icon.
+	for i := 1; i < len(hits); i++ {
+		for j := i; j > 0 && depth(hits[j]) < depth(hits[j-1]); j-- {
+			hits[j], hits[j-1] = hits[j-1], hits[j]
+		}
+	}
+	return hits
+}
+
+func depth(p string) int { return strings.Count(p, string(filepath.Separator)) }
+
+// DecodeIcon reads an icon file into an image. It handles the stdlib raster
+// formats (PNG, JPEG, GIF) directly and unwraps a .ico container, decoding
+// either the PNG a modern high-resolution favicon embeds or the classic BMP/DIB
+// bitmap older ones (Apple's among them) still ship.
+func DecodeIcon(path string) (image.Image, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	if isICO(data) {
+		img, err := decodeICO(data)
+		if err != nil {
+			return nil, fmt.Errorf("pack: decode icon %q: %w", path, err)
+		}
+		return img, nil
+	}
+	img, _, err := image.Decode(bytes.NewReader(data))
+	if err != nil {
+		return nil, fmt.Errorf("pack: decode icon %q: %w", path, err)
+	}
+	return img, nil
+}
+
+// isICO reports whether data begins with an ICONDIR header: reserved 0, type 1
+// (icon), and a non-zero image count.
+func isICO(data []byte) bool {
+	return len(data) >= 6 &&
+		data[0] == 0 && data[1] == 0 &&
+		data[2] == 1 && data[3] == 0 &&
+		(uint16(data[4])|uint16(data[5])<<8) > 0
+}
+
+var pngMagic = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}
+
+// decodeICO decodes the best entry in an .ico container. It reads the directory,
+// orders the entries largest first (a bigger source rescales to a cleaner 48x48
+// favicon), and decodes each in turn until one works: a PNG entry through the
+// stdlib, a BMP/DIB entry through decodeICOBMP. It errors only when no entry
+// decodes, so a truly garbage .ico still falls back to the default icon.
+func decodeICO(data []byte) (image.Image, error) {
+	if len(data) < 6 {
+		return nil, fmt.Errorf("pack: .ico too short")
+	}
+	count := int(binary.LittleEndian.Uint16(data[4:6]))
+	type entry struct{ w, h, off, size int }
+	var ents []entry
+	for i := 0; i < count; i++ {
+		e := 6 + i*16
+		if e+16 > len(data) {
+			break
+		}
+		w, h := int(data[e]), int(data[e+1])
+		if w == 0 {
+			w = 256
+		}
+		if h == 0 {
+			h = 256
+		}
+		size := int(binary.LittleEndian.Uint32(data[e+8 : e+12]))
+		off := int(binary.LittleEndian.Uint32(data[e+12 : e+16]))
+		if off < 0 || size <= 0 || off+size > len(data) {
+			continue
+		}
+		ents = append(ents, entry{w, h, off, size})
+	}
+	sort.SliceStable(ents, func(i, j int) bool { return ents[i].w*ents[i].h > ents[j].w*ents[j].h })
+
+	var firstErr error
+	for _, en := range ents {
+		chunk := data[en.off : en.off+en.size]
+		var (
+			img image.Image
+			err error
+		)
+		if bytes.HasPrefix(chunk, pngMagic) {
+			img, _, err = image.Decode(bytes.NewReader(chunk))
+		} else {
+			img, err = decodeICOBMP(chunk, en.w, en.h)
+		}
+		if err == nil {
+			return img, nil
+		}
+		if firstErr == nil {
+			firstErr = err
+		}
+	}
+	if firstErr == nil {
+		firstErr = fmt.Errorf("pack: .ico holds no decodable entry")
+	}
+	return nil, firstErr
+}
+
+// decodeICOBMP decodes one BMP/DIB icon entry: a BITMAPINFOHEADER, an optional
+// color table, the XOR color bitmap, then a 1-bpp AND transparency mask. The
+// header's height covers both bitmaps stacked, so the real image is half of it.
+// Rows run bottom-up and are padded to a 4-byte boundary. Only uncompressed
+// (BI_RGB) entries are handled, which covers every icon that is not a PNG.
+func decodeICOBMP(p []byte, dirW, dirH int) (image.Image, error) {
+	if len(p) < 40 {
+		return nil, fmt.Errorf("pack: ico bmp header truncated")
+	}
+	headerSize := int(binary.LittleEndian.Uint32(p[0:4]))
+	width := int(int32(binary.LittleEndian.Uint32(p[4:8])))
+	height := int(int32(binary.LittleEndian.Uint32(p[8:12])))
+	bits := int(binary.LittleEndian.Uint16(p[14:16]))
+	compression := binary.LittleEndian.Uint32(p[16:20])
+	clrUsed := int(binary.LittleEndian.Uint32(p[32:36]))
+
+	if compression != 0 {
+		return nil, fmt.Errorf("pack: ico bmp compression %d unsupported", compression)
+	}
+	height /= 2 // drop the AND-mask half to get the picture height
+	if width <= 0 || height <= 0 {
+		width, height = dirW, dirH // fall back to the directory dimensions
+	}
+	if width <= 0 || height <= 0 || width > 1024 || height > 1024 {
+		return nil, fmt.Errorf("pack: ico bmp size %dx%d unreasonable", width, height)
+	}
+
+	off := headerSize
+	if headerSize < 40 || off > len(p) {
+		off = 40
+	}
+
+	// A color table follows the header for the palettized depths (BGRA quads).
+	var palette [][4]byte
+	if bits <= 8 {
+		n := clrUsed
+		if n == 0 {
+			n = 1 << bits
+		}
+		for i := 0; i < n && off+4 <= len(p); i++ {
+			palette = append(palette, [4]byte{p[off], p[off+1], p[off+2], p[off+3]})
+			off += 4
+		}
+	}
+
+	xorStride := ((width*bits + 31) / 32) * 4
+	andStride := ((width + 31) / 32) * 4
+	xor := p[off:]
+	var and []byte
+	if andOff := xorStride * height; andOff < len(xor) {
+		and = xor[andOff:]
+	}
+
+	maskBit := func(mask []byte, rowStart, x int) bool {
+		i := rowStart + x/8
+		return i < len(mask) && (mask[i]>>(7-uint(x%8)))&1 == 1
+	}
+
+	img := image.NewNRGBA(image.Rect(0, 0, width, height))
+	anyAlpha := false
+	for y := 0; y < height; y++ {
+		srcY := height - 1 - y // rows are bottom-up
+		row := srcY * xorStride
+		andRow := srcY * andStride
+		for x := 0; x < width; x++ {
+			var r, g, b uint8
+			a := uint8(255)
+			switch bits {
+			case 32:
+				i := row + x*4
+				if i+4 > len(xor) {
+					continue
+				}
+				b, g, r, a = xor[i], xor[i+1], xor[i+2], xor[i+3]
+				if a != 0 {
+					anyAlpha = true
+				}
+			case 24:
+				i := row + x*3
+				if i+3 > len(xor) {
+					continue
+				}
+				b, g, r = xor[i], xor[i+1], xor[i+2]
+			case 8:
+				i := row + x
+				if i >= len(xor) {
+					continue
+				}
+				if c, ok := paletteAt(palette, int(xor[i])); ok {
+					b, g, r = c[0], c[1], c[2]
+				}
+			case 4:
+				i := row + x/2
+				if i >= len(xor) {
+					continue
+				}
+				idx := xor[i] >> 4
+				if x&1 == 1 {
+					idx = xor[i] & 0x0f
+				}
+				if c, ok := paletteAt(palette, int(idx)); ok {
+					b, g, r = c[0], c[1], c[2]
+				}
+			case 1:
+				i := row + x/8
+				if i >= len(xor) {
+					continue
+				}
+				bit := (xor[i] >> (7 - uint(x%8))) & 1
+				if c, ok := paletteAt(palette, int(bit)); ok {
+					b, g, r = c[0], c[1], c[2]
+				}
+			default:
+				return nil, fmt.Errorf("pack: ico bmp %d-bpp unsupported", bits)
+			}
+			if bits != 32 && maskBit(and, andRow, x) {
+				a = 0 // AND-mask transparency for the non-alpha depths
+			}
+			img.SetNRGBA(x, y, color.NRGBA{R: r, G: g, B: b, A: a})
+		}
+	}
+
+	// A 32-bpp icon whose alpha channel is entirely zero is opaque, not
+	// invisible: it leans on the AND mask instead. Reapply opacity from the mask.
+	if bits == 32 && !anyAlpha {
+		for y := 0; y < height; y++ {
+			srcY := height - 1 - y
+			andRow := srcY * andStride
+			for x := 0; x < width; x++ {
+				c := img.NRGBAAt(x, y)
+				if maskBit(and, andRow, x) {
+					c.A = 0
+				} else {
+					c.A = 255
+				}
+				img.SetNRGBA(x, y, c)
+			}
+		}
+	}
+	return img, nil
+}
+
+func paletteAt(pal [][4]byte, i int) ([4]byte, bool) {
+	if i < 0 || i >= len(pal) {
+		return [4]byte{}, false
+	}
+	return pal[i], true
+}

pack/icon_test.go

@@ -0,0 +1,212 @@
+package pack
+
+import (
+	"bytes"
+	"encoding/binary"
+	"image/color"
+	"image/png"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// pngBytes encodes a solid square as PNG, used to seed icon fixtures.
+func pngBytes(t *testing.T, px int) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	if err := png.Encode(&buf, solid(px, px, color.RGBA{R: 0x10, G: 0x80, B: 0x40, A: 0xff})); err != nil {
+		t.Fatal(err)
+	}
+	return buf.Bytes()
+}
+
+// leWrite appends v to w in little-endian, swallowing the error a bytes.Buffer
+// never returns. It keeps the .ico fixtures readable without an errcheck flag on
+// every field.
+func leWrite(w *bytes.Buffer, v any) { _ = binary.Write(w, binary.LittleEndian, v) }
+
+// icoWithPNG wraps PNG payloads in a minimal .ico container so the ICO path is
+// exercised without a binary fixture on disk.
+func icoWithPNG(pngs [][]byte) []byte {
+	var dir, body bytes.Buffer
+	put := leWrite
+	put(&dir, uint16(0))         // reserved
+	put(&dir, uint16(1))         // type: icon
+	put(&dir, uint16(len(pngs))) // count
+	offset := 6 + len(pngs)*16
+	for i, p := range pngs {
+		// width/height bytes: 0 means 256; use a distinct small size per entry.
+		dim := byte(16 * (i + 1))
+		dir.WriteByte(dim)        // width
+		dir.WriteByte(dim)        // height
+		dir.WriteByte(0)          // colours
+		dir.WriteByte(0)          // reserved
+		put(&dir, uint16(1))      // planes
+		put(&dir, uint16(32))     // bit count
+		put(&dir, uint32(len(p))) // bytes in resource
+		put(&dir, uint32(offset)) // offset
+		offset += len(p)
+		body.Write(p)
+	}
+	return append(dir.Bytes(), body.Bytes()...)
+}
+
+func TestDecodeIconPNG(t *testing.T) {
+	p := filepath.Join(t.TempDir(), "favicon.png")
+	if err := os.WriteFile(p, pngBytes(t, 64), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	img, err := DecodeIcon(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if img.Bounds().Dx() != 64 {
+		t.Errorf("decoded width = %d, want 64", img.Bounds().Dx())
+	}
+}
+
+func TestDecodeIconICOWithPNG(t *testing.T) {
+	// Two PNG entries; the decoder should pick the larger (second, 32px square).
+	ico := icoWithPNG([][]byte{pngBytes(t, 16), pngBytes(t, 32)})
+	p := filepath.Join(t.TempDir(), "favicon.ico")
+	if err := os.WriteFile(p, ico, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	img, err := DecodeIcon(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if img.Bounds().Dx() != 32 {
+		t.Errorf("decoded width = %d, want the larger 32px entry", img.Bounds().Dx())
+	}
+}
+
+func TestDecodeIconGarbageICOFails(t *testing.T) {
+	// A one-entry .ico whose payload is neither a PNG nor a complete DIB header
+	// (here four filler bytes) must error so the caller falls back to the default
+	// icon rather than crashing.
+	var ico bytes.Buffer
+	leWrite(&ico, uint16(0))
+	leWrite(&ico, uint16(1))
+	leWrite(&ico, uint16(1))
+	ico.Write([]byte{32, 32, 0, 0})
+	leWrite(&ico, uint16(1))
+	leWrite(&ico, uint16(32))
+	leWrite(&ico, uint32(4))
+	leWrite(&ico, uint32(22))
+	ico.Write([]byte{0x42, 0x4d, 0x00, 0x00}) // truncated DIB filler
+	p := filepath.Join(t.TempDir(), "favicon.ico")
+	if err := os.WriteFile(p, ico.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := DecodeIcon(p); err == nil {
+		t.Fatal("garbage .ico should fail to decode")
+	}
+}
+
+// icoWithBMP32 wraps a solid px-by-px 32-bpp BMP/DIB bitmap (the format Apple's
+// favicon.ico uses) in a single-entry .ico container, with an all-opaque AND
+// mask. It exercises the hand-written BMP path without a binary fixture.
+func icoWithBMP32(px int, c color.RGBA) []byte {
+	var dib bytes.Buffer
+	leWrite(&dib, uint32(40))    // biSize
+	leWrite(&dib, int32(px))     // biWidth
+	leWrite(&dib, int32(px*2))   // biHeight (color + mask, stacked)
+	leWrite(&dib, uint16(1))     // biPlanes
+	leWrite(&dib, uint16(32))    // biBitCount
+	leWrite(&dib, uint32(0))     // biCompression = BI_RGB
+	leWrite(&dib, uint32(0))     // biSizeImage
+	leWrite(&dib, int32(0))      // biXPelsPerMeter
+	leWrite(&dib, int32(0))      // biYPelsPerMeter
+	leWrite(&dib, uint32(0))     // biClrUsed
+	leWrite(&dib, uint32(0))     // biClrImportant
+	for i := 0; i < px*px; i++ { // XOR: BGRA, full alpha
+		dib.Write([]byte{c.B, c.G, c.R, 0xff})
+	}
+	andStride := ((px + 31) / 32) * 4
+	dib.Write(make([]byte, andStride*px)) // AND mask, all zero = opaque
+
+	var ico bytes.Buffer
+	leWrite(&ico, uint16(0))
+	leWrite(&ico, uint16(1))
+	leWrite(&ico, uint16(1))
+	ico.WriteByte(byte(px))
+	ico.WriteByte(byte(px))
+	ico.WriteByte(0)
+	ico.WriteByte(0)
+	leWrite(&ico, uint16(1))
+	leWrite(&ico, uint16(32))
+	leWrite(&ico, uint32(dib.Len()))
+	leWrite(&ico, uint32(22))
+	ico.Write(dib.Bytes())
+	return ico.Bytes()
+}
+
+func TestDecodeIconBMP32(t *testing.T) {
+	want := color.RGBA{R: 0x33, G: 0x66, B: 0x99, A: 0xff}
+	p := filepath.Join(t.TempDir(), "favicon.ico")
+	if err := os.WriteFile(p, icoWithBMP32(64, want), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	img, err := DecodeIcon(p)
+	if err != nil {
+		t.Fatalf("DecodeIcon: %v", err)
+	}
+	if img.Bounds().Dx() != 64 || img.Bounds().Dy() != 64 {
+		t.Fatalf("size = %v, want 64x64", img.Bounds())
+	}
+	r, g, b, a := img.At(10, 10).RGBA()
+	if r>>8 != 0x33 || g>>8 != 0x66 || b>>8 != 0x99 || a>>8 != 0xff {
+		t.Errorf("pixel = %02x%02x%02x%02x, want 336699ff", r>>8, g>>8, b>>8, a>>8)
+	}
+}
+
+// A BMP-only .ico now decodes end to end through FindIcon, the path that gives a
+// favicon-only site (such as developer.apple.com) its book icon.
+func TestFindIconDecodesBMPICO(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "favicon.ico"), icoWithBMP32(48, color.RGBA{R: 1, G: 2, B: 3, A: 0xff}), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	img, src, ok := FindIcon(dir)
+	if !ok {
+		t.Fatal("FindIcon should decode a BMP .ico")
+	}
+	if filepath.Base(src) != "favicon.ico" {
+		t.Errorf("src = %s, want favicon.ico", filepath.Base(src))
+	}
+	if img.Bounds().Dx() != 48 {
+		t.Errorf("width = %d, want 48", img.Bounds().Dx())
+	}
+}
+
+func TestFindIconPrefersAppleTouch(t *testing.T) {
+	dir := t.TempDir()
+	// A tiny favicon at the root and a bigger apple-touch icon a level down.
+	if err := os.WriteFile(filepath.Join(dir, "favicon.png"), pngBytes(t, 16), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	sub := filepath.Join(dir, "assets")
+	if err := os.MkdirAll(sub, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(sub, "apple-touch-icon.png"), pngBytes(t, 180), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	img, src, ok := FindIcon(dir)
+	if !ok {
+		t.Fatal("FindIcon found nothing")
+	}
+	if filepath.Base(src) != "apple-touch-icon.png" {
+		t.Errorf("picked %s, want apple-touch-icon.png", filepath.Base(src))
+	}
+	if img.Bounds().Dx() != 180 {
+		t.Errorf("width = %d, want 180", img.Bounds().Dx())
+	}
+}
+
+func TestFindIconNone(t *testing.T) {
+	if _, _, ok := FindIcon(t.TempDir()); ok {
+		t.Fatal("FindIcon should report nothing in an empty dir")
+	}
+}

pack/mime.go

@@ -0,0 +1,52 @@
+package pack
+
+import (
+	"path"
+	"strings"
+)
+
+// mimeByExt maps a lower-case file extension (with the dot) to the MIME type
+// kage records for it. Inference is by extension only, never by sniffing the
+// bytes, so the same input always yields the same output. Anything not listed
+// falls back to application/octet-stream and is stored uncompressed.
+var mimeByExt = map[string]string{
+	".html":  "text/html",
+	".htm":   "text/html",
+	".css":   "text/css",
+	".js":    "text/javascript",
+	".mjs":   "text/javascript",
+	".json":  "application/json",
+	".xml":   "application/xml",
+	".svg":   "image/svg+xml",
+	".txt":   "text/plain",
+	".png":   "image/png",
+	".jpg":   "image/jpeg",
+	".jpeg":  "image/jpeg",
+	".gif":   "image/gif",
+	".webp":  "image/webp",
+	".avif":  "image/avif",
+	".ico":   "image/x-icon",
+	".woff2": "font/woff2",
+	".woff":  "font/woff",
+	".ttf":   "font/ttf",
+	".otf":   "font/otf",
+	".eot":   "application/vnd.ms-fontobject",
+	".mp4":   "video/mp4",
+	".m4v":   "video/mp4",
+	".webm":  "video/webm",
+	".mp3":   "audio/mpeg",
+	".ogg":   "audio/ogg",
+	".pdf":   "application/pdf",
+	".zip":   "application/zip",
+	".wasm":  "application/wasm",
+}
+
+// MimeForExt returns the MIME type for a path's extension, defaulting to
+// application/octet-stream when the extension is unknown or absent.
+func MimeForExt(p string) string {
+	ext := strings.ToLower(path.Ext(p))
+	if m, ok := mimeByExt[ext]; ok {
+		return m
+	}
+	return "application/octet-stream"
+}

pack/osdetect.go

@@ -0,0 +1,57 @@
+package pack
+
+import "os"
+
+// Executable-format magic numbers, enough to tell the three desktop targets
+// apart by their first bytes. We only need the family (windows, darwin, linux),
+// not the architecture, so the smallest distinguishing prefix is plenty.
+var (
+	magicELF  = []byte{0x7f, 'E', 'L', 'F'}    // Linux, FreeBSD, and other ELF systems
+	magicPE   = []byte{'M', 'Z'}               // Windows PE/COFF (DOS stub header)
+	machOLE64 = []byte{0xcf, 0xfa, 0xed, 0xfe} // Mach-O 64-bit, little-endian (amd64, arm64)
+	machOLE32 = []byte{0xce, 0xfa, 0xed, 0xfe} // Mach-O 32-bit, little-endian
+	machOBE64 = []byte{0xfe, 0xed, 0xfa, 0xcf} // Mach-O 64-bit, big-endian
+	machOBE32 = []byte{0xfe, 0xed, 0xfa, 0xce} // Mach-O 32-bit, big-endian
+	machOFat  = []byte{0xca, 0xfe, 0xba, 0xbe} // Mach-O universal (fat) binary
+)
+
+// SniffOS reads the first bytes of an executable and returns the GOOS family it
+// was built for: "windows", "darwin", "linux", or "" when the bytes match none
+// of them. It is how pack decides whether a cross-built viewer needs a .exe
+// suffix and which run hint to print, without trusting the base's file name.
+func SniffOS(path string) string {
+	f, err := os.Open(path)
+	if err != nil {
+		return ""
+	}
+	defer func() { _ = f.Close() }()
+
+	head := make([]byte, 4)
+	if _, err := f.ReadAt(head, 0); err != nil {
+		return ""
+	}
+	switch {
+	case hasPrefix(head, magicPE):
+		return "windows"
+	case hasPrefix(head, magicELF):
+		return "linux"
+	case hasPrefix(head, machOLE64), hasPrefix(head, machOLE32),
+		hasPrefix(head, machOBE64), hasPrefix(head, machOBE32),
+		hasPrefix(head, machOFat):
+		return "darwin"
+	default:
+		return ""
+	}
+}
+
+func hasPrefix(b, prefix []byte) bool {
+	if len(b) < len(prefix) {
+		return false
+	}
+	for i := range prefix {
+		if b[i] != prefix[i] {
+			return false
+		}
+	}
+	return true
+}

pack/osdetect_test.go

@@ -0,0 +1,40 @@
+package pack
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestSniffOS(t *testing.T) {
+	cases := []struct {
+		name string
+		head []byte
+		want string
+	}{
+		{"elf", []byte{0x7f, 'E', 'L', 'F', 0x02, 0x01}, "linux"},
+		{"pe", []byte{'M', 'Z', 0x90, 0x00}, "windows"},
+		{"macho-le64", []byte{0xcf, 0xfa, 0xed, 0xfe}, "darwin"},
+		{"macho-le32", []byte{0xce, 0xfa, 0xed, 0xfe}, "darwin"},
+		{"macho-be64", []byte{0xfe, 0xed, 0xfa, 0xcf}, "darwin"},
+		{"macho-fat", []byte{0xca, 0xfe, 0xba, 0xbe}, "darwin"},
+		{"unknown", []byte{0x00, 0x01, 0x02, 0x03}, ""},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			p := filepath.Join(t.TempDir(), tc.name)
+			if err := os.WriteFile(p, tc.head, 0o644); err != nil {
+				t.Fatal(err)
+			}
+			if got := SniffOS(p); got != tc.want {
+				t.Errorf("SniffOS(%s) = %q, want %q", tc.name, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestSniffOSMissingFile(t *testing.T) {
+	if got := SniffOS(filepath.Join(t.TempDir(), "nope")); got != "" {
+		t.Errorf("SniffOS(missing) = %q, want empty", got)
+	}
+}

pack/pack_test.go

@@ -0,0 +1,356 @@
+package pack
+
+import (
+	"bytes"
+	"encoding/binary"
+	"image"
+	"image/color"
+	"image/png"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/tamnd/kage/urlx"
+	"github.com/tamnd/kage/zim"
+)
+
+// writeMirror lays down a small kage-style mirror under a temp dir and returns
+// the host dir.
+func writeMirror(t *testing.T) string {
+	t.Helper()
+	root := t.TempDir()
+	host := filepath.Join(root, "example.com")
+	files := map[string]string{
+		"index.html":                   "<!doctype html><title>Example Home</title><h1>Hi</h1>",
+		"about/index.html":             "<!doctype html><title>About</title><h1>About</h1>",
+		"_kage/example.com/x/logo.png": "\x89PNGfake",
+		"_kage/state.json":             `{"visited":[]}`, // must be skipped
+	}
+	for rel, body := range files {
+		p := filepath.Join(host, filepath.FromSlash(rel))
+		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return host
+}
+
+func TestMimeForExt(t *testing.T) {
+	cases := map[string]string{
+		"a/b/index.html":  "text/html",
+		"style.CSS":       "text/css",
+		"data.json":       "application/json",
+		"icon.svg":        "image/svg+xml",
+		"logo.png":        "image/png",
+		"photo.JPEG":      "image/jpeg",
+		"font.woff2":      "font/woff2",
+		"clip.mp4":        "video/mp4",
+		"doc.pdf":         "application/pdf",
+		"mystery":         "application/octet-stream",
+		"archive.tar.zst": "application/octet-stream",
+	}
+	for in, want := range cases {
+		if got := MimeForExt(in); got != want {
+			t.Errorf("MimeForExt(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestBuildZIMRoundTrip(t *testing.T) {
+	host := writeMirror(t)
+	out := filepath.Join(t.TempDir(), "example.zim")
+	path, size, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", Version: "test"})
+	if err != nil {
+		t.Fatalf("BuildZIM: %v", err)
+	}
+	if path != out {
+		t.Errorf("path = %q, want %q", path, out)
+	}
+	fi, err := os.Stat(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if fi.Size() != size {
+		t.Errorf("reported size %d, file is %d", size, fi.Size())
+	}
+
+	r, err := zim.Open(out)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	defer func() { _ = r.Close() }()
+
+	// Main page is the root index.
+	mp, err := r.MainPage()
+	if err != nil {
+		t.Fatalf("MainPage: %v", err)
+	}
+	if !bytes.Contains(mp.Data, []byte("Example Home")) {
+		t.Errorf("main page wrong: %.40q", mp.Data)
+	}
+	if mp.MimeType != "text/html" {
+		t.Errorf("main page mime = %q", mp.MimeType)
+	}
+
+	// Binary asset round-trips byte-for-byte.
+	logo, err := r.Get(zim.NamespaceContent, "_kage/example.com/x/logo.png")
+	if err != nil {
+		t.Fatalf("Get logo: %v", err)
+	}
+	if string(logo.Data) != "\x89PNGfake" {
+		t.Errorf("logo bytes wrong: %q", logo.Data)
+	}
+
+	// Title metadata comes from the main page's <title>.
+	title, err := r.Get(zim.NamespaceMetadata, "Title")
+	if err != nil || string(title.Data) != "Example Home" {
+		t.Errorf("M/Title = %q, %v", title.Data, err)
+	}
+
+	// state.json was skipped.
+	if _, err := r.Get(zim.NamespaceContent, urlx.DefaultReserved+"/state.json"); err == nil {
+		t.Error("state.json should not be packed")
+	}
+}
+
+func TestBuildZIMDeterministic(t *testing.T) {
+	host := writeMirror(t)
+	dir := t.TempDir()
+	a, _, err := BuildZIM(host, ZIMOptions{Out: filepath.Join(dir, "a.zim"), Date: "2026-06-14"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, _, err := BuildZIM(host, ZIMOptions{Out: filepath.Join(dir, "b.zim"), Date: "2026-06-14"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ba, _ := os.ReadFile(a)
+	bb, _ := os.ReadFile(b)
+	if !bytes.Equal(ba, bb) {
+		t.Error("same mirror produced different archives")
+	}
+}
+
+// TestBuildZIMMetadata checks the metadata zimcheck treats as mandatory: a
+// Title, Name, Language, Description, and the Illustrator_48x48@1 favicon. The
+// mirror carries a real PNG favicon so the icon path is exercised end to end.
+func TestBuildZIMMetadata(t *testing.T) {
+	host := writeMirror(t)
+
+	// Drop a 64x64 PNG favicon into the mirror so Favicon48 has something to
+	// find, decode, and rescale to 48x48.
+	icon := image.NewRGBA(image.Rect(0, 0, 64, 64))
+	for y := 0; y < 64; y++ {
+		for x := 0; x < 64; x++ {
+			icon.Set(x, y, color.RGBA{R: 0x33, G: 0x66, B: 0x99, A: 0xff})
+		}
+	}
+	var pngBuf bytes.Buffer
+	if err := png.Encode(&pngBuf, icon); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(host, "favicon.png"), pngBuf.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	out := filepath.Join(t.TempDir(), "meta.zim")
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", Version: "test"}); err != nil {
+		t.Fatalf("BuildZIM: %v", err)
+	}
+	r, err := zim.Open(out)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	defer func() { _ = r.Close() }()
+
+	wantText := map[string]string{
+		"Name":        "example.com",
+		"Language":    "eng",
+		"Description": "Offline mirror of example.com, cloned by kage.",
+	}
+	for k, want := range wantText {
+		e, err := r.Get(zim.NamespaceMetadata, k)
+		if err != nil {
+			t.Errorf("M/%s: %v", k, err)
+			continue
+		}
+		if string(e.Data) != want {
+			t.Errorf("M/%s = %q, want %q", k, e.Data, want)
+		}
+	}
+
+	// Illustrator_48x48@1 is a 48x48 PNG with an image/png MIME.
+	ill, err := r.Get(zim.NamespaceMetadata, "Illustrator_48x48@1")
+	if err != nil {
+		t.Fatalf("M/Illustrator_48x48@1: %v", err)
+	}
+	if ill.MimeType != "image/png" {
+		t.Errorf("illustrator mime = %q, want image/png", ill.MimeType)
+	}
+	img, format, err := image.Decode(bytes.NewReader(ill.Data))
+	if err != nil {
+		t.Fatalf("decode illustrator: %v", err)
+	}
+	if format != "png" {
+		t.Errorf("illustrator format = %q, want png", format)
+	}
+	if b := img.Bounds(); b.Dx() != 48 || b.Dy() != 48 {
+		t.Errorf("illustrator size = %dx%d, want 48x48", b.Dx(), b.Dy())
+	}
+
+	// A caller-supplied description overrides the host-derived default.
+	out2 := filepath.Join(t.TempDir(), "meta2.zim")
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out2, Description: "Custom blurb.", Date: "2026-06-14"}); err != nil {
+		t.Fatalf("BuildZIM: %v", err)
+	}
+	r2, err := zim.Open(out2)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	defer func() { _ = r2.Close() }()
+	if d, err := r2.Get(zim.NamespaceMetadata, "Description"); err != nil || string(d.Data) != "Custom blurb." {
+		t.Errorf("M/Description = %q, %v; want %q", d.Data, err, "Custom blurb.")
+	}
+}
+
+func TestPickMainPage(t *testing.T) {
+	cases := []struct {
+		in   []string
+		want string
+	}{
+		{[]string{"a/index.html", "index.html", "b.html"}, "index.html"},
+		{[]string{"z/deep/p.html", "top.html", "a/p.html"}, "top.html"},
+		{[]string{"b/x.html", "a/x.html"}, "a/x.html"}, // same depth, lexical
+		{nil, ""},
+	}
+	for _, c := range cases {
+		if got := pickMainPage(c.in); got != c.want {
+			t.Errorf("pickMainPage(%v) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+// TestBinaryTrailerRoundTrip exercises the BuildBinary append contract and the
+// trailer it leaves, without depending on os.Executable: it appends a ZIM to a
+// fake base, reads the trailer back the way Embedded does, and serves the
+// recovered archive.
+func TestBinaryTrailerRoundTrip(t *testing.T) {
+	host := writeMirror(t)
+	zbytes, err := BuildZIMBytes(host, ZIMOptions{Date: "2026-06-14"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dir := t.TempDir()
+	base := filepath.Join(dir, "fakekage")
+	baseBytes := bytes.Repeat([]byte("BASE"), 64) // stand-in for a kage binary
+	if err := os.WriteFile(base, baseBytes, 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	out := filepath.Join(dir, "viewer")
+	_, total, err := BuildBinary(zbytes, BinaryOptions{Out: out, Base: base})
+	if err != nil {
+		t.Fatalf("BuildBinary: %v", err)
+	}
+	if total != int64(len(baseBytes)+len(zbytes)+trailerLen) {
+		t.Errorf("total %d, want %d", total, len(baseBytes)+len(zbytes)+trailerLen)
+	}
+
+	f, err := os.Open(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = f.Close() }()
+	fi, _ := f.Stat()
+	end := fi.Size()
+
+	tr := make([]byte, trailerLen)
+	if _, err := f.ReadAt(tr, end-int64(trailerLen)); err != nil {
+		t.Fatal(err)
+	}
+	if string(tr[:8]) != trailerMagic || string(tr[trailerLen-8:]) != trailerMagic {
+		t.Fatal("trailer magic missing")
+	}
+	zlen := int64(binary.LittleEndian.Uint64(tr[8:16]))
+	if zlen != int64(len(zbytes)) {
+		t.Errorf("trailer length %d, want %d", zlen, len(zbytes))
+	}
+	start := end - int64(trailerLen) - zlen
+	if start != int64(len(baseBytes)) {
+		t.Errorf("archive start %d, want %d", start, len(baseBytes))
+	}
+
+	sec := io.NewSectionReader(f, start, zlen)
+	r, err := zim.NewReader(sec, zlen)
+	if err != nil {
+		t.Fatalf("reopen appended zim: %v", err)
+	}
+	mp, err := r.MainPage()
+	if err != nil || !bytes.Contains(mp.Data, []byte("Example Home")) {
+		t.Errorf("recovered main page wrong: %.40q (%v)", mp.Data, err)
+	}
+}
+
+func TestHandler(t *testing.T) {
+	host := writeMirror(t)
+	out := filepath.Join(t.TempDir(), "h.zim")
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14"}); err != nil {
+		t.Fatal(err)
+	}
+	r, err := zim.Open(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = r.Close() }()
+
+	srv := httptest.NewServer(Handler(r))
+	defer srv.Close()
+
+	get := func(p string) (int, string) {
+		resp, err := http.Get(srv.URL + p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() { _ = resp.Body.Close() }()
+		b, _ := io.ReadAll(resp.Body)
+		return resp.StatusCode, string(b)
+	}
+
+	if code, body := get("/"); code != 200 || !bytes.Contains([]byte(body), []byte("Example Home")) {
+		t.Errorf("GET / = %d %.30q", code, body)
+	}
+
+	// "/" must redirect to the main page's canonical content path, not serve its
+	// bytes in place, so the page's mirror-relative asset URLs resolve against the
+	// right base instead of 404ing.
+	noFollow := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
+		return http.ErrUseLastResponse
+	}}
+	resp, err := noFollow.Get(srv.URL + "/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusFound {
+		t.Errorf("GET / status = %d, want 302", resp.StatusCode)
+	}
+	if loc := resp.Header.Get("Location"); loc != "/index.html" {
+		t.Errorf("GET / Location = %q, want %q", loc, "/index.html")
+	}
+	if code, _ := get("/about/index.html"); code != 200 {
+		t.Errorf("GET /about/index.html = %d", code)
+	}
+	if code, _ := get("/" + urlx.DefaultReserved + "/state.json"); code != 404 {
+		t.Errorf("GET state.json = %d, want 404", code)
+	}
+	if code, _ := get("/missing.html"); code != 404 {
+		t.Errorf("GET missing = %d, want 404", code)
+	}
+}

pack/serve.go

@@ -0,0 +1,55 @@
+package pack
+
+import (
+	"errors"
+	"net/http"
+	"strings"
+
+	"github.com/tamnd/kage/zim"
+)
+
+// Handler serves a ZIM archive over HTTP. "/" redirects to the archive's main
+// page; "/a/b.png" maps to the C/a/b.png content entry. Because the saved HTML's
+// links are mirror-relative paths, and those are exactly the C urls, a click in a
+// served page hits the right entry with no rewriting. A miss is a plain 404.
+func Handler(r *zim.Reader) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		p := strings.TrimPrefix(req.URL.Path, "/")
+		if p == "" {
+			// The main page's saved HTML carries mirror-relative asset URLs
+			// (../_kage/...) computed for its own nested location, so serving its
+			// bytes at "/" would resolve them against the wrong base and 404 the
+			// page's CSS and images. Redirect to the page's canonical content path
+			// instead, the way the archive's W/mainPage redirect does, so the
+			// browser resolves those relative URLs correctly.
+			if ns, url, ok := r.MainPageRef(); ok && ns == zim.NamespaceContent {
+				http.Redirect(w, req, "/"+url, http.StatusFound)
+				return
+			}
+			blob, err := r.MainPage()
+			if err != nil {
+				http.NotFound(w, req)
+				return
+			}
+			serveBlob(w, blob)
+			return
+		}
+		blob, err := r.Get(zim.NamespaceContent, p)
+		if errors.Is(err, zim.ErrNotFound) {
+			http.NotFound(w, req)
+			return
+		}
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		serveBlob(w, blob)
+	})
+}
+
+func serveBlob(w http.ResponseWriter, b zim.Blob) {
+	if b.MimeType != "" {
+		w.Header().Set("Content-Type", b.MimeType)
+	}
+	_, _ = w.Write(b.Data)
+}

pack/zim.go

@@ -0,0 +1,306 @@
+// Package pack turns a kage mirror on disk into a distributable artifact: a ZIM
+// archive, or a self-contained executable that serves the mirror offline. It is
+// the only pack-side package that touches the filesystem and the running
+// executable; the byte-level format work lives in the zim package.
+package pack
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"golang.org/x/net/html"
+
+	"github.com/tamnd/kage/urlx"
+	"github.com/tamnd/kage/zim"
+)
+
+// ZIMOptions controls how a mirror is packed into a ZIM archive. Date is passed
+// in from the CLI boundary rather than read from the clock, so the zim and pack
+// packages stay pure and packing the same mirror twice is byte-identical.
+type ZIMOptions struct {
+	Out         string // output path (default <mirror-base>.zim)
+	NoCompress  bool   // store every cluster raw (code 1)
+	Title       string // overrides M/Title
+	Description string // M/Description
+	Language    string // M/Language (default "eng")
+	Date        string // M/Date, e.g. "2026-06-14"
+	Version     string // kage version, recorded as M/Scraper
+
+	// CachePath, when set, points at a content-addressed cluster cache sidecar.
+	// Compression of unchanged clusters is reused from it, and the cache is
+	// rewritten after a successful pack. Empty disables the cache. It has no
+	// effect with NoCompress, where nothing is compressed.
+	CachePath string
+	// Stats, when non-nil, is filled with how many clusters were reused from the
+	// cache versus compressed fresh. It lets the caller report incremental gains
+	// without changing the function's return signature.
+	Stats *PackStats
+}
+
+// PackStats reports how a pack reused cached compression. ClustersReused is the
+// number of clusters whose compressed bytes came straight from the cache;
+// ClustersCompressed is the number that were zstd-compressed this run.
+type PackStats struct {
+	ClustersReused     int
+	ClustersCompressed int
+}
+
+// BuildZIM walks mirrorDir, turns every file into a C/ content entry, infers the
+// MIME from the extension, picks a main page, adds M/ metadata and a W/mainPage
+// redirect, and writes a .zim to opts.Out. It returns the output path and the
+// number of bytes written.
+func BuildZIM(mirrorDir string, opts ZIMOptions) (string, int64, error) {
+	w, cache, err := buildWriter(mirrorDir, opts)
+	if err != nil {
+		return "", 0, err
+	}
+	out := opts.Out
+	if out == "" {
+		out = filepath.Base(mirrorDir) + ".zim"
+	}
+	f, err := os.Create(out)
+	if err != nil {
+		return "", 0, err
+	}
+	bw := bufio.NewWriter(f)
+	n, err := w.WriteTo(bw)
+	if err != nil {
+		_ = f.Close()
+		return out, n, err
+	}
+	if err := bw.Flush(); err != nil {
+		_ = f.Close()
+		return out, n, err
+	}
+	if err := f.Close(); err != nil {
+		return out, n, err
+	}
+	if err := persistCache(opts.CachePath, cache); err != nil {
+		return out, n, err
+	}
+	fillStats(opts.Stats, cache)
+	return out, n, nil
+}
+
+// BuildZIMBytes is the buffer-returning sibling of BuildZIM: it runs the same
+// walk and returns the archive in memory, which the binary path appends to a
+// base executable without writing the ZIM to disk first.
+func BuildZIMBytes(mirrorDir string, opts ZIMOptions) ([]byte, error) {
+	w, cache, err := buildWriter(mirrorDir, opts)
+	if err != nil {
+		return nil, err
+	}
+	var buf bytes.Buffer
+	if _, err := w.WriteTo(&buf); err != nil {
+		return nil, err
+	}
+	if err := persistCache(opts.CachePath, cache); err != nil {
+		return nil, err
+	}
+	fillStats(opts.Stats, cache)
+	return buf.Bytes(), nil
+}
+
+// persistCache writes the cluster cache back to its sidecar after a successful
+// pack. It is a no-op when caching is off.
+func persistCache(path string, c *clusterCache) error {
+	if c == nil || path == "" {
+		return nil
+	}
+	return c.save(path)
+}
+
+// fillStats copies the cache's reuse counters into the caller's PackStats when
+// both are present, so the CLI can report incremental gains.
+func fillStats(dst *PackStats, c *clusterCache) {
+	if dst == nil || c == nil {
+		return
+	}
+	dst.ClustersReused = c.reused
+	dst.ClustersCompressed = c.compressed
+}
+
+// buildWriter does the shared work of both BuildZIM and BuildZIMBytes: it loads
+// every file under mirrorDir into a zim.Writer with metadata and a main page.
+func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, *clusterCache, error) {
+	info, err := os.Stat(mirrorDir)
+	if err != nil {
+		return nil, nil, err
+	}
+	if !info.IsDir() {
+		return nil, nil, fmt.Errorf("pack: %q is not a directory", mirrorDir)
+	}
+
+	w := zim.NewWriter()
+	if opts.NoCompress {
+		w.SetNoCompress(true)
+	}
+
+	// The cluster cache only helps when clusters are compressed; with NoCompress
+	// there is nothing to reuse, so it is skipped.
+	var cache *clusterCache
+	if opts.CachePath != "" && !opts.NoCompress {
+		cache = loadClusterCache(opts.CachePath)
+		w.SetCompress(cache.Compress)
+	}
+
+	skip := urlx.DefaultReserved + "/state.json"
+	var htmlPages []string
+	counts := map[string]int{}
+
+	walkErr := filepath.WalkDir(mirrorDir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		rel := slashRel(mirrorDir, p)
+		if rel == skip {
+			return nil
+		}
+		data, err := os.ReadFile(p)
+		if err != nil {
+			return err
+		}
+		mime := MimeForExt(rel)
+		if mime == "text/html" {
+			htmlPages = append(htmlPages, rel)
+		}
+		counts[mime]++
+		w.AddContent(zim.NamespaceContent, rel, "", mime, data)
+		return nil
+	})
+	if walkErr != nil {
+		return nil, nil, walkErr
+	}
+
+	main := pickMainPage(htmlPages)
+	if main != "" {
+		w.SetMainPage(zim.NamespaceContent, main)
+		w.AddRedirect(zim.NamespaceWellKnown, "mainPage", "", zim.NamespaceContent, main)
+	}
+
+	host := filepath.Base(mirrorDir)
+	title := firstNonEmpty(opts.Title, htmlTitleOf(mirrorDir, main), host)
+	w.AddMetadata("Title", title)
+	w.AddMetadata("Name", host)
+	w.AddMetadata("Language", firstNonEmpty(opts.Language, "eng"))
+	// Description is mandatory metadata in the ZIM spec, so it is always written:
+	// the caller's text when given, otherwise a line derived from the host.
+	w.AddMetadata("Description", firstNonEmpty(opts.Description, "Offline mirror of "+host+", cloned by kage."))
+	w.AddMetadata("Creator", "kage")
+	w.AddMetadata("Publisher", "kage")
+	if opts.Date != "" {
+		w.AddMetadata("Date", opts.Date)
+	}
+	w.AddMetadata("Scraper", strings.TrimSpace("kage "+opts.Version))
+	w.AddMetadata("Source", host)
+	w.AddMetadata("Counter", counterString(counts))
+	// Illustrator_48x48@1 is the 48x48 PNG favicon Kiwix shows as the archive's
+	// icon. When the mirror has no usable icon the archive ships without one.
+	if png, ok := Favicon48(mirrorDir); ok {
+		w.AddMetadataBytes("Illustrator_48x48@1", "image/png", png)
+	}
+	return w, cache, nil
+}
+
+// pickMainPage chooses the archive's entry point: the root index if present,
+// else the shallowest HTML page, ties broken lexicographically for determinism.
+// It returns "" when the mirror has no HTML at all.
+func pickMainPage(htmlPages []string) string {
+	for _, p := range htmlPages {
+		if p == "index.html" {
+			return p
+		}
+	}
+	sorted := append([]string(nil), htmlPages...)
+	sort.Slice(sorted, func(i, j int) bool {
+		di, dj := strings.Count(sorted[i], "/"), strings.Count(sorted[j], "/")
+		if di != dj {
+			return di < dj
+		}
+		return sorted[i] < sorted[j]
+	})
+	if len(sorted) > 0 {
+		return sorted[0]
+	}
+	return ""
+}
+
+// htmlTitleOf reads the main page off disk and returns its <title>, or "" if
+// there is no main page or no title.
+func htmlTitleOf(mirrorDir, mainURL string) string {
+	if mainURL == "" {
+		return ""
+	}
+	f, err := os.Open(filepath.Join(mirrorDir, filepath.FromSlash(mainURL)))
+	if err != nil {
+		return ""
+	}
+	defer func() { _ = f.Close() }()
+	doc, err := html.Parse(f)
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(findTitle(doc))
+}
+
+// findTitle returns the text of the first <title> element in depth-first order.
+func findTitle(n *html.Node) string {
+	if n.Type == html.ElementNode && n.Data == "title" {
+		var b strings.Builder
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			if c.Type == html.TextNode {
+				b.WriteString(c.Data)
+			}
+		}
+		return b.String()
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		if t := findTitle(c); t != "" {
+			return t
+		}
+	}
+	return ""
+}
+
+// counterString renders the M/Counter value Kiwix uses for stats: a
+// semicolon-separated list of mime=count pairs, sorted for determinism.
+func counterString(counts map[string]int) string {
+	mimes := make([]string, 0, len(counts))
+	for m := range counts {
+		mimes = append(mimes, m)
+	}
+	sort.Strings(mimes)
+	parts := make([]string, len(mimes))
+	for i, m := range mimes {
+		parts[i] = fmt.Sprintf("%s=%d", m, counts[m])
+	}
+	return strings.Join(parts, ";")
+}
+
+// slashRel returns p relative to root using forward slashes, the form ZIM urls
+// take regardless of the host filesystem separator.
+func slashRel(root, p string) string {
+	rel, err := filepath.Rel(root, p)
+	if err != nil {
+		rel = p
+	}
+	return filepath.ToSlash(rel)
+}
+
+func firstNonEmpty(vals ...string) string {
+	for _, v := range vals {
+		if v != "" {
+			return v
+		}
+	}
+	return ""
+}

sanitize/sanitize.go

@@ -2,9 +2,11 @@
 // the saved page is inert: a photograph, not a program.
 //
 // It parses with golang.org/x/net/html, walks the tree, and deletes scripts,
-// event handlers, javascript: URLs, and the dead preconnect/preload hints that
-// mean nothing offline — while leaving styles, images, fonts, forms, and all
-// semantic markup untouched so the layout survives intact.
+// event handlers, javascript: URLs, downlevel IE conditional comments (which
+// can smuggle a <script> past an element-only walk), and the dead
+// preconnect/preload hints that mean nothing offline — while leaving styles,
+// images, fonts, forms, and all semantic markup untouched so the layout
+// survives intact.
 package sanitize
 
 import (
@@ -31,13 +33,15 @@ type Options struct {
 
 // Report counts what was removed, for the run summary and for tests.
 type Report struct {
-	ScriptsRemoved     int
-	HandlersRemoved    int
-	NoscriptRemoved    int
-	NoscriptUnwrapped  int
-	JSURLsNeutralized  int
-	MetaRefreshRemoved int
-	DeadLinksRemoved   int
+	ScriptsRemoved      int
+	HandlersRemoved     int
+	NoscriptRemoved     int
+	NoscriptUnwrapped   int
+	JSURLsNeutralized   int
+	MetaRefreshRemoved  int
+	DeadLinksRemoved    int
+	CondCommentsRemoved int
+	CharsetAdded        bool
 }
 
 // jsURLAttrs are attributes whose value may be a javascript: URL.
@@ -67,6 +71,7 @@ func Strip(doc []byte, opts Options) ([]byte, Report, error) {
 func CleanTree(root *html.Node, opts Options) Report {
 	var rep Report
 	clean(root, opts, &rep)
+	rep.CharsetAdded = ensureCharset(root)
 	if opts.Banner != "" {
 		insertBanner(root, opts.Banner)
 	}
@@ -78,6 +83,18 @@ func clean(n *html.Node, opts Options, rep *Report) {
 	var next *html.Node
 	for c := n.FirstChild; c != nil; c = next {
 		next = c.NextSibling
+		if c.Type == html.CommentNode {
+			// A downlevel IE conditional comment (<!--[if lt IE 9]>...<![endif]-->)
+			// parses as one comment whose data holds raw markup — a <script src>
+			// among it. The element walk never sees that script, so drop the whole
+			// comment. Downlevel-revealed content lives in sibling nodes, not the
+			// comment data, so it is untouched.
+			if isConditionalComment(c.Data) {
+				n.RemoveChild(c)
+				rep.CondCommentsRemoved++
+			}
+			continue
+		}
 		if c.Type == html.ElementNode {
 			switch c.DataAtom {
 			case atom.Script:
@@ -175,6 +192,19 @@ func isDeadLink(n *html.Node) bool {
 	return false
 }
 
+// isConditionalComment reports whether a comment's data is a downlevel IE
+// conditional-comment marker. Both the downlevel-hidden form (the whole
+// "[if lt IE 9]>...<![endif]" in one comment) and the two markers of the
+// downlevel-revealed form ("[if gte IE 9]><!" and "<![endif]") match, so the
+// markers are stripped while any revealed content, which sits in sibling
+// nodes, stays.
+func isConditionalComment(data string) bool {
+	d := strings.TrimSpace(data)
+	return strings.HasPrefix(d, "[if") ||
+		strings.HasPrefix(d, "<![endif]") ||
+		strings.HasPrefix(d, "[endif]")
+}
+
 // unwrapNoscript replaces a <noscript> with its content. Because x/net/html
 // parses noscript content as raw text (scripting enabled), the text is
 // re-parsed as a fragment in the parent's context and spliced in before the
@@ -199,6 +229,68 @@ func unwrapNoscript(parent, ns *html.Node) {
 	parent.RemoveChild(ns)
 }
 
+// ensureCharset guarantees the document declares UTF-8, inserting a
+// <meta charset="utf-8"> at the top of <head> when none is present, and reports
+// whether it added one. kage renders every saved page as UTF-8, but a source
+// that set its charset only in the HTTP Content-Type header, with no <meta>
+// charset in the markup, loses that signal once the page is a standalone file.
+// A reader then serving the bytes without a charset falls back to its locale
+// encoding and mojibakes every multibyte character (curly quotes, dashes, a
+// non-breaking space). Declaring the charset in the markup makes the page
+// self-describing in any reader, kage's own viewer and Kiwix alike.
+func ensureCharset(root *html.Node) bool {
+	head := findElement(root, atom.Head)
+	if head == nil {
+		return false
+	}
+	if hasCharsetMeta(head) {
+		return false
+	}
+	meta := &html.Node{
+		Type:     html.ElementNode,
+		Data:     "meta",
+		DataAtom: atom.Meta,
+		Attr:     []html.Attribute{{Key: "charset", Val: "utf-8"}},
+	}
+	// The declaration must precede any content for a reader to honour it, so it
+	// goes first in <head>.
+	head.InsertBefore(meta, head.FirstChild)
+	return true
+}
+
+// hasCharsetMeta reports whether head already declares a character encoding,
+// either as <meta charset="..."> or the older <meta http-equiv="Content-Type"
+// content="...; charset=...">.
+func hasCharsetMeta(head *html.Node) bool {
+	for c := head.FirstChild; c != nil; c = c.NextSibling {
+		if c.Type != html.ElementNode || c.DataAtom != atom.Meta {
+			continue
+		}
+		if attr(c, "charset") != "" {
+			return true
+		}
+		if strings.EqualFold(attr(c, "http-equiv"), "content-type") &&
+			strings.Contains(strings.ToLower(attr(c, "content")), "charset=") {
+			return true
+		}
+	}
+	return false
+}
+
+// findElement returns the first element node of the given atom in document
+// order, or nil if none exists.
+func findElement(n *html.Node, a atom.Atom) *html.Node {
+	if n.Type == html.ElementNode && n.DataAtom == a {
+		return n
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		if found := findElement(c, a); found != nil {
+			return found
+		}
+	}
+	return nil
+}
+
 // insertBanner prepends an HTML comment to the document.
 func insertBanner(root *html.Node, text string) {
 	c := &html.Node{Type: html.CommentNode, Data: " " + text + " "}

sanitize/sanitize_test.go

@@ -107,6 +107,48 @@ func TestKeepNoscriptUnwraps(t *testing.T) {
 	}
 }
 
+func TestConditionalCommentScriptRemoved(t *testing.T) {
+	// A downlevel-hidden IE conditional comment hides a <script src> inside a
+	// single comment node, where an element-only walk never reaches it.
+	in := `<html><head>
+<!--[if lt IE 9]><script src="//oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script><![endif]-->
+</head><body><p>real</p></body></html>`
+	out, rep, err := Strip([]byte(in), Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := string(out)
+	if strings.Contains(s, "<script") || strings.Contains(s, "html5shiv") {
+		t.Errorf("conditional-comment script survived:\n%s", s)
+	}
+	if strings.Contains(s, "[if lt IE 9]") {
+		t.Errorf("conditional comment survived:\n%s", s)
+	}
+	if rep.CondCommentsRemoved != 1 {
+		t.Errorf("CondCommentsRemoved = %d, want 1", rep.CondCommentsRemoved)
+	}
+	if !strings.Contains(s, "<p>real</p>") {
+		t.Errorf("real content must survive:\n%s", s)
+	}
+}
+
+func TestConditionalCommentRevealedContentKept(t *testing.T) {
+	// The downlevel-revealed form shows its content to non-IE browsers; the
+	// content lives in sibling nodes, so only the two markers are stripped.
+	in := `<html><body><!--[if gte IE 9]><!--><span class="modern">keep me</span><!--<![endif]--></body></html>`
+	out, _, err := Strip([]byte(in), Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := string(out)
+	if !strings.Contains(s, `<span class="modern">keep me</span>`) {
+		t.Errorf("revealed content was dropped:\n%s", s)
+	}
+	if strings.Contains(s, "[if") || strings.Contains(s, "<![endif]") {
+		t.Errorf("conditional markers survived:\n%s", s)
+	}
+}
+
 func TestKeepMetaRefreshPlain(t *testing.T) {
 	in := `<html><head><meta http-equiv="refresh" content="5;url=/next"></head><body></body></html>`
 	out, _, err := Strip([]byte(in), Options{KeepMetaRefresh: true})
@@ -124,3 +166,53 @@ func TestKeepMetaRefreshPlain(t *testing.T) {
 		t.Error("JS-target meta refresh must be removed regardless")
 	}
 }
+
+func TestCharsetAddedWhenMissing(t *testing.T) {
+	// A page whose source declared its charset only in the HTTP header has no
+	// <meta charset>. The saved file must gain one so a reader does not fall back
+	// to its locale encoding and mojibake the UTF-8 text.
+	in := `<html><head><title>Quotes</title></head><body><p>` +
+		"“curly” — café</p></body></html>"
+	out, rep, err := Strip([]byte(in), Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !rep.CharsetAdded {
+		t.Error("CharsetAdded = false, want true")
+	}
+	s := string(out)
+	if !strings.Contains(strings.ToLower(s), `<meta charset="utf-8"/>`) {
+		t.Errorf("expected an injected meta charset:\n%s", s)
+	}
+	// It must sit at the very start of <head>, before any content.
+	headIdx := strings.Index(s, "<head>")
+	metaIdx := strings.Index(strings.ToLower(s), "<meta charset")
+	titleIdx := strings.Index(s, "<title>")
+	if headIdx >= metaIdx || metaIdx >= titleIdx {
+		t.Errorf("meta charset must come first in head (head=%d meta=%d title=%d)", headIdx, metaIdx, titleIdx)
+	}
+	// The original bytes are preserved as UTF-8.
+	if !strings.Contains(s, "café") {
+		t.Error("UTF-8 content should be preserved")
+	}
+}
+
+func TestCharsetNotDuplicated(t *testing.T) {
+	// A page that already declares a charset, in either form, is left alone.
+	cases := []string{
+		`<html><head><meta charset="utf-8"><title>x</title></head><body></body></html>`,
+		`<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>x</title></head><body></body></html>`,
+	}
+	for _, in := range cases {
+		out, rep, err := Strip([]byte(in), Options{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if rep.CharsetAdded {
+			t.Errorf("CharsetAdded = true for a page that already declares one:\n%s", in)
+		}
+		if n := strings.Count(strings.ToLower(string(out)), "charset"); n != 1 {
+			t.Errorf("charset count = %d, want 1:\n%s", n, out)
+		}
+	}
+}

urlx/urlx.go

@@ -19,6 +19,8 @@ import (
 	"net/url"
 	"path"
 	"strings"
+
+	"golang.org/x/net/publicsuffix"
 )
 
 // Kind distinguishes a crawlable page from a downloadable asset; the two map to
@@ -142,9 +144,59 @@ func canonical(u *url.URL, root bool) *url.URL {
 		}
 		c.Path = cleaned
 	}
+	c.RawQuery = safeRawQuery(c.RawQuery)
 	return &c
 }
 
+// safeRawQuery percent-encodes the characters a query string is not allowed to
+// carry on the wire (a space, a control byte, a non-ASCII byte) while leaving
+// everything already legal untouched, including existing %XX escapes and the
+// query sub-delimiters a cache-buster relies on (& = , ; etc.). It exists
+// because real sites emit hrefs with raw spaces in the query, e.g. a CSS link
+// busted with a date string like "?Thursday, 26-Feb-2026 16:26:41 UTC"; a
+// browser encodes the spaces before requesting, but url.Parse keeps RawQuery
+// verbatim, so without this the request line is malformed and the server
+// answers 400. Re-encoding here, on the canonical URL, fixes both the fetch and
+// the on-disk key in one place.
+func safeRawQuery(raw string) string {
+	if raw == "" {
+		return ""
+	}
+	var b strings.Builder
+	for i := 0; i < len(raw); i++ {
+		ch := raw[i]
+		switch {
+		case ch == '%' && i+2 < len(raw) && isHex(raw[i+1]) && isHex(raw[i+2]):
+			// Already a percent-escape; copy it through unchanged.
+			b.WriteString(raw[i : i+3])
+			i += 2
+		case queryByteAllowed(ch):
+			b.WriteByte(ch)
+		default:
+			b.WriteByte('%')
+			const hexDigits = "0123456789ABCDEF"
+			b.WriteByte(hexDigits[ch>>4])
+			b.WriteByte(hexDigits[ch&0x0f])
+		}
+	}
+	return b.String()
+}
+
+// queryByteAllowed reports whether ch may appear literally in a URL query.
+// It is the RFC 3986 query grammar (pchar / "/" / "?"), which keeps the
+// sub-delims that structure a query and rejects spaces, quotes, and controls.
+func queryByteAllowed(ch byte) bool {
+	switch {
+	case ch >= 'a' && ch <= 'z', ch >= 'A' && ch <= 'Z', ch >= '0' && ch <= '9':
+		return true
+	}
+	return strings.IndexByte("-._~!$&'()*+,;=:@/?", ch) >= 0
+}
+
+func isHex(ch byte) bool {
+	return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')
+}
+
 // Key is the canonical string form used to dedup pages and assets.
 func Key(u *url.URL) string { return u.String() }
 
@@ -168,6 +220,34 @@ func SameSite(seed, u *url.URL, allowSub bool) bool {
 	return false
 }
 
+// SameRegistrableDomain reports whether u shares the seed's registrable domain
+// (its eTLD+1). It is looser than SameSite: developer.apple.com, www.apple.com,
+// and images.apple.com all fold to apple.com and count as same-domain, while a
+// separate brand like cdn-apple.com or an unrelated third party like
+// ec.europa.eu does not. It is how kage decides whether an asset host is "the
+// site's own" without listing every subdomain a CDN might use. When either host
+// has no registrable domain (an IP, an oddball TLD), it falls back to an exact
+// host match so the decision stays conservative.
+func SameRegistrableDomain(seed, u *url.URL) bool {
+	sh, uh := seed.Hostname(), u.Hostname()
+	if sh == uh {
+		return true
+	}
+	sd, err1 := publicsuffix.EffectiveTLDPlusOne(sh)
+	ud, err2 := publicsuffix.EffectiveTLDPlusOne(uh)
+	if err1 != nil || err2 != nil {
+		return false
+	}
+	return sd == ud
+}
+
+// Ext returns the lowercased file extension of a URL's last path segment,
+// including the leading dot (".pdf", ".mp4"), or "" when there is none. It
+// ignores the query string, so "/a/clip.mp4?v=2" reports ".mp4".
+func Ext(u *url.URL) string {
+	return strings.ToLower(path.Ext(lastSegment(u.Path)))
+}
+
 // InScope reports whether u should be crawled as a page given the seed and cfg.
 func InScope(seed, u *url.URL, cfg ScopeConfig) bool {
 	if !SameSite(seed, u, cfg.IncludeSubdomains) {

urlx/urlx_test.go

@@ -61,6 +61,12 @@ func TestNormalize(t *testing.T) {
 		{"//cdn.io/x.css", "https://cdn.io/x.css", false},
 		{"HTTPS://Other.COM/Y", "https://other.com/Y", false},
 		{"sub/", "https://ex.com/docs/sub/", false},
+		// A query busted with a raw date string must come back with its spaces
+		// percent-encoded so the request line is valid, while the commas and
+		// colons a query legally carries stay as they are.
+		{"a.css?Thursday, 26-Feb-2026 16:26:41 UTC", "https://ex.com/docs/a.css?Thursday,%2026-Feb-2026%2016:26:41%20UTC", false},
+		{"b.css?v=1&t=a b", "https://ex.com/docs/b.css?v=1&t=a%20b", false},
+		{"c.css?x=%20done", "https://ex.com/docs/c.css?x=%20done", false},
 		{"#top", "", true},
 		{"javascript:void(0)", "", true},
 		{"mailto:a@b.com", "", true},
@@ -110,6 +116,40 @@ func TestInScope(t *testing.T) {
 	}
 }
 
+func TestSameRegistrableDomain(t *testing.T) {
+	seed := mustParse(t, "https://developer.apple.com/")
+	cases := map[string]bool{
+		"https://developer.apple.com/x": true,
+		"https://www.apple.com/x":       true,
+		"https://images.apple.com/x":    true,
+		"https://apple.com/x":           true,
+		"https://cdn-apple.com/x":       false,
+		"https://ec.europa.eu/x":        false,
+		"https://mmbiz.qpic.cn/x":       false,
+	}
+	for u, want := range cases {
+		if got := SameRegistrableDomain(seed, mustParse(t, u)); got != want {
+			t.Errorf("SameRegistrableDomain(%q) = %v, want %v", u, got, want)
+		}
+	}
+}
+
+func TestExt(t *testing.T) {
+	cases := map[string]string{
+		"https://ex.com/a/clip.mp4":     ".mp4",
+		"https://ex.com/a/clip.MP4?v=2": ".mp4",
+		"https://ex.com/style.css":      ".css",
+		"https://ex.com/docs/":          "",
+		"https://ex.com/docs":           "",
+		"https://ex.com/Xcode_15.0.dmg": ".dmg",
+	}
+	for u, want := range cases {
+		if got := Ext(mustParse(t, u)); got != want {
+			t.Errorf("Ext(%q) = %q, want %q", u, got, want)
+		}
+	}
+}
+
 func TestLikelyPage(t *testing.T) {
 	cases := map[string]bool{
 		"https://ex.com/docs":      true,

viewer/browser.go

@@ -0,0 +1,41 @@
+//go:build !webview
+
+package viewer
+
+import (
+	"context"
+	"os/exec"
+	"runtime"
+)
+
+// Native is false in the default pure-Go build: there is no native window, so
+// the viewer hands the URL to the system browser.
+const Native = false
+
+// LockMainThread is a no-op without a native UI to pin to the main thread.
+func LockMainThread() {}
+
+// Show opens the system browser at o.URL when o.Browser is set, then blocks
+// until the context is cancelled (Ctrl-C), leaving the caller's HTTP server up
+// in the meantime. Launching the browser is best-effort; a failure is ignored
+// because the URL has already been printed for the user to open by hand.
+func Show(ctx context.Context, o Options) error {
+	if o.Browser {
+		_ = openInBrowser(o.URL)
+	}
+	<-ctx.Done()
+	return nil
+}
+
+func openInBrowser(url string) error {
+	var cmd *exec.Cmd
+	switch runtime.GOOS {
+	case "darwin":
+		cmd = exec.Command("open", url)
+	case "windows":
+		cmd = exec.Command("rundll32", "url.dll,FileProtocolHandler", url)
+	default:
+		cmd = exec.Command("xdg-open", url)
+	}
+	return cmd.Start()
+}

viewer/browser_test.go

@@ -0,0 +1,37 @@
+//go:build !webview
+
+package viewer
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestNativeIsFalseInDefaultBuild(t *testing.T) {
+	if Native {
+		t.Fatal("Native should be false without the webview build tag")
+	}
+}
+
+func TestLockMainThreadIsNoop(t *testing.T) {
+	// Must not panic; there is no native UI to pin to.
+	LockMainThread()
+}
+
+func TestShowReturnsWhenContextCancelled(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	// Browser:false so no system browser is launched during the test.
+	go func() { done <- Show(ctx, Options{URL: "http://127.0.0.1:0", Browser: false}) }()
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("Show returned error: %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("Show did not return after context cancellation")
+	}
+}

viewer/viewer.go

@@ -0,0 +1,24 @@
+// Package viewer presents a served site to the user. It has two
+// implementations chosen at build time: by default (pure Go, CGO_ENABLED=0) it
+// opens the system browser, and with the "webview" build tag (which needs cgo)
+// it opens a native window backed by the operating system's WebView, so a
+// packed kage binary feels like a standalone app rather than a browser tab.
+//
+// Both builds expose the same three symbols: Native, LockMainThread, and Show.
+// The caller starts an HTTP server, then calls Show on the main goroutine; Show
+// blocks until the window is closed (native) or the context is cancelled
+// (browser), at which point the caller shuts the server down.
+package viewer
+
+// Options configures a viewer window.
+type Options struct {
+	Title string // window title; the archive's M/Title, falling back to "kage"
+	URL   string // local URL the server is listening on
+	// Browser, in the default build, opens the system browser. The native build
+	// ignores it and always shows its own window.
+	Browser bool
+}
+
+// Native reports whether this build opens a native window (webview tag) or
+// falls back to the system browser. Show and LockMainThread are defined in the
+// per-build files browser.go and webview.go.

viewer/webview.go

@@ -0,0 +1,53 @@
+//go:build webview
+
+package viewer
+
+import (
+	"context"
+	"runtime"
+
+	webview "github.com/webview/webview_go"
+)
+
+// Native is true in the webview build: Show opens a real window backed by the
+// operating system's WebView (WKWebView on macOS, WebView2 on Windows,
+// WebKitGTK on Linux), so a packed kage feels like a standalone app.
+//
+// This build needs cgo and links the platform WebView, so it is opt-in
+// (-tags webview) and kept out of the default CGO_ENABLED=0 release pipeline.
+const Native = true
+
+// LockMainThread pins the calling goroutine to its OS thread. main calls it
+// first thing, while the main goroutine is still on the process's initial
+// thread, because the macOS WebView must be driven from that thread.
+func LockMainThread() { runtime.LockOSThread() }
+
+// Show opens a native window pointed at o.URL and runs the UI event loop on the
+// calling (main) goroutine, blocking until the window is closed. A cancelled
+// context terminates the loop too, so Ctrl-C still shuts the viewer down. The
+// o.Browser flag is ignored: the whole point of this build is the native window.
+func Show(ctx context.Context, o Options) error {
+	w := webview.New(false)
+	defer w.Destroy()
+
+	title := o.Title
+	if title == "" {
+		title = "kage"
+	}
+	w.SetTitle(title)
+	w.SetSize(1024, 768, webview.HintNone)
+	w.Navigate(o.URL)
+
+	done := make(chan struct{})
+	go func() {
+		select {
+		case <-ctx.Done():
+			w.Dispatch(func() { w.Terminate() })
+		case <-done:
+		}
+	}()
+
+	w.Run()
+	close(done)
+	return nil
+}

zim/codec.go

@@ -0,0 +1,54 @@
+package zim
+
+import (
+	"sync"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+// A single shared zstd codec. Both EncodeAll and DecodeAll are safe for
+// concurrent use, so one encoder and one decoder serve the whole process.
+var (
+	zstdOnce sync.Once
+	zstdEnc  *zstd.Encoder
+	zstdDec  *zstd.Decoder
+)
+
+func initZstd() {
+	zstdOnce.Do(func() {
+		zstdEnc, _ = zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
+		zstdDec, _ = zstd.NewReader(nil)
+	})
+}
+
+func zstdEncode(p []byte) []byte {
+	initZstd()
+	return zstdEnc.EncodeAll(p, nil)
+}
+
+// Compress zstd-compresses p with the exact codec the writer uses for its
+// clusters. It is exported so a caller can cache cluster compression across
+// packs and feed the result back through Writer.SetCompress; a cached cluster is
+// then byte-for-byte what a fresh compression would have produced.
+func Compress(p []byte) []byte { return zstdEncode(p) }
+
+func zstdDecode(p []byte) ([]byte, error) {
+	initZstd()
+	return zstdDec.DecodeAll(p, nil)
+}
+
+// isTextMime reports whether content of this MIME type is worth compressing.
+// Already-compressed media (images, fonts, audio, video, archives) is stored
+// uncompressed so we do not burn CPU inflating it by a few bytes.
+func isTextMime(mime string) bool {
+	switch mime {
+	case "application/json", "application/xml", "application/javascript",
+		"application/x-javascript":
+		return true
+	}
+	if len(mime) >= 5 && mime[:5] == "text/" {
+		return true
+	}
+	// Any structured-XML type: application/rss+xml, image/svg+xml, ...
+	return len(mime) >= 4 && mime[len(mime)-4:] == "+xml"
+}

zim/format.go

@@ -0,0 +1,115 @@
+// Package zim reads and writes the ZIM offline-archive format, the open
+// single-file container that Kiwix uses to ship offline content. kage uses it
+// to pack a cloned mirror into one indexable, compressed file that a reader can
+// random-access without unpacking.
+//
+// The package is pure: no network, no clock, no global state beyond a lazily
+// built zstd codec. A ZIM file is laid out as a fixed header, a MIME-type list,
+// three pointer lists (URL, title, cluster), a run of directory entries, a run
+// of clusters that hold the content, and a trailing MD5. Every cross-reference
+// is an absolute file position recorded in the header, so the writer assigns
+// positions in one pass and emits bytes in a second. All integers are
+// little-endian.
+//
+// We write the new namespace scheme (minor version 1): all content lives under
+// the single 'C' namespace, metadata under 'M', and a 'W/mainPage' redirect
+// points at the entry point. Reading handles redirects and both offset widths.
+package zim
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+// Magic is the ZIM header magic number, the first four bytes of every file.
+const Magic uint32 = 0x44D495A // 72173914
+
+const (
+	majorVersion uint16 = 5
+	minorVersion uint16 = 1 // single 'C' content namespace
+	headerLen           = 80
+)
+
+// Namespaces in the new (minor version 1) scheme.
+const (
+	NamespaceContent   byte = 'C' // pages and assets
+	NamespaceMetadata  byte = 'M' // M/Title, M/Date, ...
+	NamespaceWellKnown byte = 'W' // W/mainPage redirect
+)
+
+// Compression codes carried in the low nibble of a cluster's info byte.
+const (
+	compNone uint8 = 1 // stored, no compression
+	compXZ   uint8 = 4 // xz / LZMA2 (read-only support)
+	compZstd uint8 = 5 // zstd (what we write for text)
+
+	extendedFlag uint8 = 0x10 // bit 4: cluster offsets are uint64, not uint32
+)
+
+// Sentinels stored in a directory entry's mimetype field to mark non-content
+// entries. A redirect reuses the cluster slot to hold its target's URL index.
+const (
+	redirectEntry   uint16 = 0xffff
+	linkTargetEntry uint16 = 0xfffe
+	deletedEntry    uint16 = 0xfffd
+)
+
+// noMainPage is the mainPage/layoutPage value meaning "none".
+const noMainPage uint32 = 0xffffffff
+
+// header is the 80-byte ZIM header.
+type header struct {
+	uuid          [16]byte
+	articleCount  uint32
+	clusterCount  uint32
+	urlPtrPos     uint64
+	titlePtrPos   uint64
+	clusterPtrPos uint64
+	mimeListPos   uint64
+	mainPage      uint32
+	layoutPage    uint32
+	checksumPos   uint64
+}
+
+// marshal encodes the header to its 80 wire bytes.
+func (h header) marshal() []byte {
+	b := make([]byte, headerLen)
+	le := binary.LittleEndian
+	le.PutUint32(b[0:], Magic)
+	le.PutUint16(b[4:], majorVersion)
+	le.PutUint16(b[6:], minorVersion)
+	copy(b[8:24], h.uuid[:])
+	le.PutUint32(b[24:], h.articleCount)
+	le.PutUint32(b[28:], h.clusterCount)
+	le.PutUint64(b[32:], h.urlPtrPos)
+	le.PutUint64(b[40:], h.titlePtrPos)
+	le.PutUint64(b[48:], h.clusterPtrPos)
+	le.PutUint64(b[56:], h.mimeListPos)
+	le.PutUint32(b[64:], h.mainPage)
+	le.PutUint32(b[68:], h.layoutPage)
+	le.PutUint64(b[72:], h.checksumPos)
+	return b
+}
+
+// parseHeader decodes and validates an 80-byte header.
+func parseHeader(b []byte) (header, error) {
+	var h header
+	if len(b) < headerLen {
+		return h, fmt.Errorf("zim: short header: %d bytes", len(b))
+	}
+	le := binary.LittleEndian
+	if le.Uint32(b[0:]) != Magic {
+		return h, fmt.Errorf("zim: bad magic, not a ZIM file")
+	}
+	copy(h.uuid[:], b[8:24])
+	h.articleCount = le.Uint32(b[24:])
+	h.clusterCount = le.Uint32(b[28:])
+	h.urlPtrPos = le.Uint64(b[32:])
+	h.titlePtrPos = le.Uint64(b[40:])
+	h.clusterPtrPos = le.Uint64(b[48:])
+	h.mimeListPos = le.Uint64(b[56:])
+	h.mainPage = le.Uint32(b[64:])
+	h.layoutPage = le.Uint32(b[68:])
+	h.checksumPos = le.Uint64(b[72:])
+	return h, nil
+}

zim/reader.go

@@ -0,0 +1,400 @@
+package zim
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+)
+
+// ErrNotFound is returned by Get when no entry matches the namespace and url.
+// Callers (such as the HTTP handler) test for it with errors.Is to map a miss
+// to a 404.
+var ErrNotFound = errors.New("zim: not found")
+
+// Reader provides random access to a ZIM file's entries. Open one with Open or
+// NewReader, then look entries up by namespace and url, or fetch the main page.
+// Decompressed clusters are cached so repeated reads from one cluster are cheap.
+type Reader struct {
+	ra     io.ReaderAt
+	closer io.Closer
+	size   int64
+	hdr    header
+	mimes  []string
+
+	mu            sync.Mutex
+	cache         map[uint32][]byte // cluster index -> decompressed data section
+	cacheExtended map[uint32]bool   // cluster index -> uint64-offset cluster
+}
+
+// Blob is the result of a lookup: the resolved entry's bytes and metadata.
+type Blob struct {
+	Namespace byte
+	URL       string
+	Title     string
+	MimeType  string
+	Data      []byte
+}
+
+// Open opens a ZIM file on disk. Close the returned reader when done.
+func Open(path string) (*Reader, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	fi, err := f.Stat()
+	if err != nil {
+		_ = f.Close()
+		return nil, err
+	}
+	r, err := NewReader(f, fi.Size())
+	if err != nil {
+		_ = f.Close()
+		return nil, err
+	}
+	r.closer = f
+	return r, nil
+}
+
+// NewReader reads the header and MIME list from ra, which must hold size bytes.
+func NewReader(ra io.ReaderAt, size int64) (*Reader, error) {
+	r := &Reader{ra: ra, size: size, cache: map[uint32][]byte{}}
+	hb, err := r.at(0, headerLen)
+	if err != nil {
+		return nil, fmt.Errorf("zim: read header: %w", err)
+	}
+	r.hdr, err = parseHeader(hb)
+	if err != nil {
+		return nil, err
+	}
+	if r.hdr.mimeListPos > r.hdr.urlPtrPos || r.hdr.urlPtrPos > uint64(size) {
+		return nil, fmt.Errorf("zim: inconsistent header offsets")
+	}
+	mb, err := r.at(r.hdr.mimeListPos, int(r.hdr.urlPtrPos-r.hdr.mimeListPos))
+	if err != nil {
+		return nil, fmt.Errorf("zim: read mime list: %w", err)
+	}
+	for _, part := range bytes.Split(mb, []byte{0}) {
+		if len(part) == 0 {
+			break
+		}
+		r.mimes = append(r.mimes, string(part))
+	}
+	return r, nil
+}
+
+// Close releases the underlying file, if Open created one.
+func (r *Reader) Close() error {
+	if r.closer != nil {
+		return r.closer.Close()
+	}
+	return nil
+}
+
+// Count returns the number of directory entries.
+func (r *Reader) Count() uint32 { return r.hdr.articleCount }
+
+// MimeTypes returns the archive's MIME-type list.
+func (r *Reader) MimeTypes() []string { return r.mimes }
+
+// MainPage returns the archive's entry point, or an error if none is set.
+func (r *Reader) MainPage() (Blob, error) {
+	if r.hdr.mainPage == noMainPage {
+		return Blob{}, fmt.Errorf("zim: no main page")
+	}
+	return r.blobAtIndex(r.hdr.mainPage, 0)
+}
+
+// Entry is one directory entry as stored, returned by EntryAt. A redirect keeps
+// Data nil and names its target in RedirectNamespace/RedirectURL; any other
+// entry carries its bytes in Data and its type in MimeType. Unlike Get, EntryAt
+// does not follow redirects, so a caller can round-trip every entry, the
+// redirects included.
+type Entry struct {
+	Namespace         byte
+	URL               string
+	Title             string
+	MimeType          string
+	Redirect          bool
+	RedirectNamespace byte
+	RedirectURL       string
+	Data              []byte
+}
+
+// EntryAt returns the directory entry at idx, where 0 <= idx < Count, in the
+// archive's URL order. It is the iteration counterpart to Get: it exposes every
+// entry exactly as stored, including metadata and redirects, which is what an
+// exporter needs to reproduce the archive.
+func (r *Reader) EntryAt(idx uint32) (Entry, error) {
+	d, err := r.direntAtIndex(idx)
+	if err != nil {
+		return Entry{}, err
+	}
+	e := Entry{Namespace: d.namespace, URL: d.url, Title: d.title}
+	if d.redirect {
+		e.Redirect = true
+		td, err := r.direntAtIndex(d.targetIndex)
+		if err != nil {
+			return Entry{}, fmt.Errorf("zim: redirect target of %c/%s: %w", d.namespace, d.url, err)
+		}
+		e.RedirectNamespace = td.namespace
+		e.RedirectURL = td.url
+		return e, nil
+	}
+	data, err := r.blobData(d.cluster, d.blob)
+	if err != nil {
+		return Entry{}, err
+	}
+	if int(d.mimeIdx) < len(r.mimes) {
+		e.MimeType = r.mimes[d.mimeIdx]
+	}
+	e.Data = data
+	return e, nil
+}
+
+// MainPageRef returns the namespace and url of the archive's entry point and
+// whether one is set, so an exporter can record which entry is the main page
+// without following the W/mainPage redirect.
+func (r *Reader) MainPageRef() (byte, string, bool) {
+	if r.hdr.mainPage == noMainPage {
+		return 0, "", false
+	}
+	d, err := r.direntAtIndex(r.hdr.mainPage)
+	if err != nil {
+		return 0, "", false
+	}
+	return d.namespace, d.url, true
+}
+
+// Get resolves the entry at (namespace, url), following one or more redirects.
+func (r *Reader) Get(namespace byte, url string) (Blob, error) {
+	target := key(namespace, url)
+	lo, hi := uint32(0), r.hdr.articleCount
+	for lo < hi {
+		mid := lo + (hi-lo)/2
+		d, err := r.direntAtIndex(mid)
+		if err != nil {
+			return Blob{}, err
+		}
+		switch k := key(d.namespace, d.url); {
+		case k < target:
+			lo = mid + 1
+		case k > target:
+			hi = mid
+		default:
+			return r.blobAtIndex(mid, 0)
+		}
+	}
+	return Blob{}, fmt.Errorf("%w: %c/%s", ErrNotFound, namespace, url)
+}
+
+const maxRedirectHops = 16
+
+func (r *Reader) blobAtIndex(idx uint32, hop int) (Blob, error) {
+	if hop > maxRedirectHops {
+		return Blob{}, fmt.Errorf("zim: redirect loop")
+	}
+	d, err := r.direntAtIndex(idx)
+	if err != nil {
+		return Blob{}, err
+	}
+	if d.redirect {
+		return r.blobAtIndex(d.targetIndex, hop+1)
+	}
+	data, err := r.blobData(d.cluster, d.blob)
+	if err != nil {
+		return Blob{}, err
+	}
+	mime := ""
+	if int(d.mimeIdx) < len(r.mimes) {
+		mime = r.mimes[d.mimeIdx]
+	}
+	return Blob{Namespace: d.namespace, URL: d.url, Title: d.title, MimeType: mime, Data: data}, nil
+}
+
+type dirent struct {
+	mimeIdx     uint16
+	namespace   byte
+	url, title  string
+	cluster     uint32
+	blob        uint32
+	redirect    bool
+	targetIndex uint32
+}
+
+func (r *Reader) direntAtIndex(idx uint32) (dirent, error) {
+	pb, err := r.at(r.hdr.urlPtrPos+8*uint64(idx), 8)
+	if err != nil {
+		return dirent{}, err
+	}
+	return r.direntAt(binary.LittleEndian.Uint64(pb))
+}
+
+func (r *Reader) direntAt(off uint64) (dirent, error) {
+	// Read a window large enough for the fixed head plus url and title; grow if
+	// either string is not terminated within it.
+	window := 512
+	for {
+		b, err := r.at(off, window)
+		if err != nil && len(b) == 0 {
+			return dirent{}, err
+		}
+		var d dirent
+		le := binary.LittleEndian
+		d.mimeIdx = le.Uint16(b[0:])
+		d.namespace = b[3]
+		var p int
+		if d.mimeIdx == redirectEntry {
+			d.redirect = true
+			d.targetIndex = le.Uint32(b[8:])
+			p = 12
+		} else {
+			d.cluster = le.Uint32(b[8:])
+			d.blob = le.Uint32(b[12:])
+			p = 16
+		}
+		url, n1, ok := readCString(b, p)
+		if !ok {
+			if window >= 1<<20 || off+uint64(window) >= uint64(r.size) {
+				return dirent{}, fmt.Errorf("zim: unterminated url at %d", off)
+			}
+			window *= 4
+			continue
+		}
+		title, _, ok := readCString(b, n1)
+		if !ok {
+			if window >= 1<<20 || off+uint64(window) >= uint64(r.size) {
+				return dirent{}, fmt.Errorf("zim: unterminated title at %d", off)
+			}
+			window *= 4
+			continue
+		}
+		d.url, d.title = url, title
+		return d, nil
+	}
+}
+
+// blobData returns one blob's bytes, decompressing and caching its cluster.
+func (r *Reader) blobData(cluster, blob uint32) ([]byte, error) {
+	data, extended, err := r.clusterData(cluster)
+	if err != nil {
+		return nil, err
+	}
+	w := uint32(4)
+	if extended {
+		w = 8
+	}
+	need := int((blob + 2) * w)
+	if need > len(data) {
+		return nil, fmt.Errorf("zim: blob %d out of range in cluster %d", blob, cluster)
+	}
+	o0 := readUint(data[blob*w:], w)
+	o1 := readUint(data[(blob+1)*w:], w)
+	if o0 > o1 || int(o1) > len(data) {
+		return nil, fmt.Errorf("zim: bad blob offsets in cluster %d", cluster)
+	}
+	out := make([]byte, o1-o0)
+	copy(out, data[o0:o1])
+	return out, nil
+}
+
+func (r *Reader) clusterData(cluster uint32) (data []byte, extended bool, err error) {
+	r.mu.Lock()
+	if c, ok := r.cache[cluster]; ok {
+		r.mu.Unlock()
+		// extended-ness is recoverable from the info byte, but the cache stores
+		// already-decoded data whose offsets we re-read with the recorded width.
+		return c, r.cacheExtended[cluster], nil
+	}
+	r.mu.Unlock()
+
+	start, err := r.clusterOffset(cluster)
+	if err != nil {
+		return nil, false, err
+	}
+	end := r.hdr.checksumPos
+	if cluster+1 < r.hdr.clusterCount {
+		if end, err = r.clusterOffset(cluster + 1); err != nil {
+			return nil, false, err
+		}
+	}
+	if start >= end || end > uint64(r.size) {
+		return nil, false, fmt.Errorf("zim: bad cluster bounds for %d", cluster)
+	}
+	raw, err := r.at(start, int(end-start))
+	if err != nil {
+		return nil, false, err
+	}
+	info := raw[0]
+	comp := info & 0x0f
+	extended = info&extendedFlag != 0
+	body := raw[1:]
+	switch comp {
+	case compNone:
+		data = body
+	case compZstd:
+		if data, err = zstdDecode(body); err != nil {
+			return nil, false, fmt.Errorf("zim: zstd cluster %d: %w", cluster, err)
+		}
+	case compXZ:
+		return nil, false, fmt.Errorf("zim: xz clusters are not supported for reading")
+	default:
+		return nil, false, fmt.Errorf("zim: unknown compression %d in cluster %d", comp, cluster)
+	}
+	r.mu.Lock()
+	r.cache[cluster] = data
+	if r.cacheExtended == nil {
+		r.cacheExtended = map[uint32]bool{}
+	}
+	r.cacheExtended[cluster] = extended
+	r.mu.Unlock()
+	return data, extended, nil
+}
+
+func (r *Reader) clusterOffset(cluster uint32) (uint64, error) {
+	b, err := r.at(r.hdr.clusterPtrPos+8*uint64(cluster), 8)
+	if err != nil {
+		return 0, err
+	}
+	return binary.LittleEndian.Uint64(b), nil
+}
+
+// at reads n bytes at off, clamped to the file size.
+func (r *Reader) at(off uint64, n int) ([]byte, error) {
+	if n < 0 {
+		return nil, fmt.Errorf("zim: negative read length")
+	}
+	if off > uint64(r.size) {
+		return nil, io.EOF
+	}
+	if off+uint64(n) > uint64(r.size) {
+		n = int(uint64(r.size) - off)
+	}
+	b := make([]byte, n)
+	if n == 0 {
+		return b, nil
+	}
+	_, err := r.ra.ReadAt(b, int64(off))
+	return b, err
+}
+
+func readCString(b []byte, start int) (string, int, bool) {
+	if start > len(b) {
+		return "", start, false
+	}
+	i := bytes.IndexByte(b[start:], 0)
+	if i < 0 {
+		return "", start, false
+	}
+	return string(b[start : start+i]), start + i + 1, true
+}
+
+func readUint(b []byte, width uint32) uint32 {
+	if width == 8 {
+		return uint32(binary.LittleEndian.Uint64(b))
+	}
+	return binary.LittleEndian.Uint32(b)
+}

zim/writer.go

@@ -0,0 +1,421 @@
+package zim
+
+import (
+	"crypto/md5"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"sort"
+)
+
+// maxClusterContent caps how much blob content accumulates in one cluster
+// before a new one is started, balancing compression ratio against the cost of
+// decompressing a whole cluster to read one small blob.
+const maxClusterContent = 2 << 20 // 2 MiB
+
+// Writer accumulates entries and serialises them as a ZIM file. Build it with
+// NewWriter, add content/redirects/metadata, optionally set a main page, then
+// call WriteTo. The writer holds entries in memory; a kage mirror comfortably
+// fits, and packing is a one-shot batch job.
+type Writer struct {
+	entries    []*entry
+	byKey      map[string]*entry
+	mainKey    string
+	noCompress bool
+	// compress turns a cluster's uncompressed data section into its stored bytes.
+	// It defaults to the built-in zstd codec; a caller can replace it with a
+	// caching compressor so a re-pack reuses the compression of unchanged
+	// clusters instead of running zstd again.
+	compress func([]byte) []byte
+}
+
+type entry struct {
+	namespace byte
+	url       string
+	title     string
+	mime      string
+	data      []byte
+
+	redirect  bool
+	targetKey string // "<ns><url>" of the redirect target
+
+	// assigned during planning
+	mimeIdx     uint16
+	cluster     uint32
+	blob        uint32
+	targetIndex uint32
+	urlIndex    uint32
+	position    uint64
+}
+
+func key(ns byte, url string) string { return string(ns) + url }
+
+// NewWriter returns an empty Writer.
+func NewWriter() *Writer {
+	return &Writer{byKey: map[string]*entry{}, compress: zstdEncode}
+}
+
+// SetNoCompress stores every cluster uncompressed. Useful when the input is
+// already compressed or when a reader without zstd must open the file.
+func (w *Writer) SetNoCompress(v bool) { w.noCompress = v }
+
+// SetCompress replaces the cluster compressor. The function must return
+// zstd-compressed bytes (the writer marks those clusters as zstd), so a caching
+// wrapper can short-circuit unchanged clusters while still producing a valid,
+// byte-identical archive. A nil function restores the built-in codec.
+func (w *Writer) SetCompress(f func([]byte) []byte) {
+	if f == nil {
+		f = zstdEncode
+	}
+	w.compress = f
+}
+
+// AddContent adds a content entry. A later add with the same namespace and url
+// replaces the earlier one. An empty title defaults to the url.
+func (w *Writer) AddContent(namespace byte, url, title, mime string, data []byte) {
+	if title == "" {
+		title = url
+	}
+	if mime == "" {
+		mime = "application/octet-stream"
+	}
+	w.put(&entry{namespace: namespace, url: url, title: title, mime: mime, data: data})
+}
+
+// AddMetadata adds an 'M' namespace text entry, e.g. AddMetadata("Title", "...").
+func (w *Writer) AddMetadata(name, value string) {
+	w.put(&entry{namespace: NamespaceMetadata, url: name, title: name, mime: "text/plain", data: []byte(value)})
+}
+
+// AddMetadataBytes adds an 'M' namespace entry with an explicit MIME, for binary
+// metadata such as Illustrator_48x48@1, the 48x48 PNG favicon Kiwix shows as the
+// archive's icon.
+func (w *Writer) AddMetadataBytes(name, mime string, data []byte) {
+	if mime == "" {
+		mime = "application/octet-stream"
+	}
+	w.put(&entry{namespace: NamespaceMetadata, url: name, title: name, mime: mime, data: data})
+}
+
+// AddRedirect adds a redirect from (namespace,url) to (targetNamespace,targetURL).
+func (w *Writer) AddRedirect(namespace byte, url, title string, targetNamespace byte, targetURL string) {
+	if title == "" {
+		title = url
+	}
+	w.put(&entry{namespace: namespace, url: url, title: title, redirect: true, targetKey: key(targetNamespace, targetURL)})
+}
+
+// SetMainPage marks an entry as the archive's entry point.
+func (w *Writer) SetMainPage(namespace byte, url string) { w.mainKey = key(namespace, url) }
+
+func (w *Writer) put(e *entry) {
+	k := key(e.namespace, e.url)
+	if old, ok := w.byKey[k]; ok {
+		*old = *e // replace in place, keep slice order
+		return
+	}
+	w.byKey[k] = e
+	w.entries = append(w.entries, e)
+}
+
+// plan holds the prebuilt sections of the file, ready to emit in order.
+type plan struct {
+	hdr         header
+	mimeList    []byte
+	urlPtrs     []byte
+	titlePtrs   []byte
+	clusterPtrs []byte
+	dirents     [][]byte // URL order
+	clusters    [][]byte
+}
+
+// WriteTo serialises the archive to out and returns the number of bytes written.
+func (w *Writer) WriteTo(out io.Writer) (int64, error) {
+	p, err := w.buildPlan()
+	if err != nil {
+		return 0, err
+	}
+	sum := md5.New()
+	mw := io.MultiWriter(out, sum)
+	var n int64
+	write := func(b []byte) error {
+		m, err := mw.Write(b)
+		n += int64(m)
+		return err
+	}
+	for _, section := range append([][]byte{
+		p.hdr.marshal(), p.mimeList, p.urlPtrs, p.titlePtrs, p.clusterPtrs,
+	}, append(p.dirents, p.clusters...)...) {
+		if err := write(section); err != nil {
+			return n, err
+		}
+	}
+	// The MD5 covers everything before it and is not itself hashed.
+	m, err := out.Write(sum.Sum(nil))
+	n += int64(m)
+	return n, err
+}
+
+func (w *Writer) buildPlan() (plan, error) {
+	var p plan
+	// 1. URL order: sort by <namespace><url>, assign indices.
+	ents := make([]*entry, len(w.entries))
+	copy(ents, w.entries)
+	sort.Slice(ents, func(i, j int) bool {
+		return key(ents[i].namespace, ents[i].url) < key(ents[j].namespace, ents[j].url)
+	})
+	index := make(map[string]uint32, len(ents))
+	for i, e := range ents {
+		e.urlIndex = uint32(i)
+		index[key(e.namespace, e.url)] = uint32(i)
+	}
+
+	// 2. Resolve redirect targets.
+	for _, e := range ents {
+		if !e.redirect {
+			continue
+		}
+		ti, ok := index[e.targetKey]
+		if !ok {
+			return p, fmt.Errorf("zim: redirect %q points at missing target %q", key(e.namespace, e.url), e.targetKey)
+		}
+		e.targetIndex = ti
+	}
+
+	// 3. MIME list (first-seen order over content entries).
+	var mimes []string
+	mimeIndex := map[string]uint16{}
+	for _, e := range ents {
+		if e.redirect {
+			continue
+		}
+		if _, ok := mimeIndex[e.mime]; !ok {
+			mimeIndex[e.mime] = uint16(len(mimes))
+			mimes = append(mimes, e.mime)
+		}
+		e.mimeIdx = mimeIndex[e.mime]
+	}
+	p.mimeList = encodeMimeList(mimes)
+
+	// 4. Cluster packing: split text vs binary, cap each cluster, assign blobs.
+	clusters := w.packClusters(ents)
+	p.clusters = make([][]byte, len(clusters))
+	for i, c := range clusters {
+		p.clusters[i] = c.encode(w.noCompress, w.compress)
+	}
+
+	// 5. Directory entry bytes (URL order).
+	p.dirents = make([][]byte, len(ents))
+	for i, e := range ents {
+		p.dirents[i] = e.encodeDirent()
+	}
+
+	// 6. Layout: assign absolute positions.
+	count := uint32(len(ents))
+	pos := uint64(headerLen)
+	mimeListPos := pos
+	pos += uint64(len(p.mimeList))
+	urlPtrPos := pos
+	pos += 8 * uint64(count)
+	titlePtrPos := pos
+	pos += 4 * uint64(count)
+	clusterPtrPos := pos
+	pos += 8 * uint64(len(p.clusters))
+	for i, e := range ents {
+		e.position = pos
+		pos += uint64(len(p.dirents[i]))
+	}
+	clusterPos := make([]uint64, len(p.clusters))
+	for i := range p.clusters {
+		clusterPos[i] = pos
+		pos += uint64(len(p.clusters[i]))
+	}
+	checksumPos := pos
+
+	// 7. Pointer lists.
+	p.urlPtrs = make([]byte, 8*count)
+	for i, e := range ents {
+		binary.LittleEndian.PutUint64(p.urlPtrs[8*i:], e.position)
+	}
+	p.clusterPtrs = make([]byte, 8*len(clusterPos))
+	for i, cp := range clusterPos {
+		binary.LittleEndian.PutUint64(p.clusterPtrs[8*i:], cp)
+	}
+	p.titlePtrs = encodeTitlePtrs(ents)
+
+	// 8. Header.
+	p.hdr = header{
+		uuid:          deriveUUID(ents),
+		articleCount:  count,
+		clusterCount:  uint32(len(p.clusters)),
+		urlPtrPos:     urlPtrPos,
+		titlePtrPos:   titlePtrPos,
+		clusterPtrPos: clusterPtrPos,
+		mimeListPos:   mimeListPos,
+		mainPage:      noMainPage,
+		layoutPage:    noMainPage,
+		checksumPos:   checksumPos,
+	}
+	if w.mainKey != "" {
+		if mi, ok := index[w.mainKey]; ok {
+			p.hdr.mainPage = mi
+		}
+	}
+	return p, nil
+}
+
+// clusterBuf accumulates blobs destined for one cluster.
+type clusterBuf struct {
+	comp  uint8
+	blobs [][]byte
+	size  int
+}
+
+func (w *Writer) packClusters(ents []*entry) []*clusterBuf {
+	var clusters []*clusterBuf
+	var curText, curBin *clusterBuf
+
+	closeIf := func(c **clusterBuf) {
+		if *c != nil && (*c).size >= maxClusterContent {
+			*c = nil
+		}
+	}
+	add := func(cur **clusterBuf, comp uint8, e *entry) {
+		if *cur == nil {
+			*cur = &clusterBuf{comp: comp}
+			clusters = append(clusters, *cur)
+		}
+		c := *cur
+		e.cluster = uint32(indexOf(clusters, c))
+		e.blob = uint32(len(c.blobs))
+		c.blobs = append(c.blobs, e.data)
+		c.size += len(e.data)
+	}
+
+	for _, e := range ents {
+		if e.redirect {
+			continue
+		}
+		if isTextMime(e.mime) {
+			add(&curText, compZstd, e)
+			closeIf(&curText)
+		} else {
+			add(&curBin, compNone, e)
+			closeIf(&curBin)
+		}
+	}
+	return clusters
+}
+
+func indexOf(cs []*clusterBuf, c *clusterBuf) int {
+	for i := range cs {
+		if cs[i] == c {
+			return i
+		}
+	}
+	return -1
+}
+
+// encode renders a cluster: an info byte followed by the (optionally zstd)
+// data section, which is an offset table of (N+1) uint32 values then the N
+// concatenated blobs. Offsets are relative to the start of the data section.
+func (c *clusterBuf) encode(noCompress bool, compress func([]byte) []byte) []byte {
+	if compress == nil {
+		compress = zstdEncode
+	}
+	n := len(c.blobs)
+	tableLen := 4 * (n + 1)
+	total := tableLen
+	for _, b := range c.blobs {
+		total += len(b)
+	}
+	data := make([]byte, tableLen, total)
+	off := uint32(tableLen)
+	binary.LittleEndian.PutUint32(data[0:], off)
+	for i, b := range c.blobs {
+		off += uint32(len(b))
+		binary.LittleEndian.PutUint32(data[4*(i+1):], off)
+	}
+	for _, b := range c.blobs {
+		data = append(data, b...)
+	}
+
+	comp := c.comp
+	if noCompress {
+		comp = compNone
+	}
+	payload := data
+	if comp == compZstd {
+		payload = compress(data)
+	} else {
+		comp = compNone
+	}
+	out := make([]byte, 0, len(payload)+1)
+	out = append(out, comp) // non-extended: bit 4 clear, uint32 offsets
+	return append(out, payload...)
+}
+
+func (e *entry) encodeDirent() []byte {
+	le := binary.LittleEndian
+	var head []byte
+	if e.redirect {
+		head = make([]byte, 12)
+		le.PutUint16(head[0:], redirectEntry)
+		head[3] = e.namespace
+		le.PutUint32(head[8:], e.targetIndex)
+	} else {
+		head = make([]byte, 16)
+		le.PutUint16(head[0:], e.mimeIdx)
+		head[3] = e.namespace
+		le.PutUint32(head[8:], e.cluster)
+		le.PutUint32(head[12:], e.blob)
+	}
+	out := append(head, e.url...)
+	out = append(out, 0)
+	out = append(out, e.title...)
+	return append(out, 0)
+}
+
+func encodeMimeList(mimes []string) []byte {
+	var b []byte
+	for _, m := range mimes {
+		b = append(b, m...)
+		b = append(b, 0)
+	}
+	return append(b, 0) // terminating empty string
+}
+
+func encodeTitlePtrs(ents []*entry) []byte {
+	order := make([]*entry, len(ents))
+	copy(order, ents)
+	sort.Slice(order, func(i, j int) bool {
+		ti := string(order[i].namespace) + order[i].title
+		tj := string(order[j].namespace) + order[j].title
+		if ti != tj {
+			return ti < tj
+		}
+		return order[i].urlIndex < order[j].urlIndex
+	})
+	b := make([]byte, 4*len(order))
+	for i, e := range order {
+		binary.LittleEndian.PutUint32(b[4*i:], e.urlIndex)
+	}
+	return b
+}
+
+// deriveUUID makes the file deterministic: identical input yields an identical
+// archive. It hashes every entry's key and content, so repacking the same
+// mirror is idempotent and diffable.
+func deriveUUID(ents []*entry) [16]byte {
+	h := md5.New()
+	var n [8]byte
+	for _, e := range ents {
+		h.Write([]byte(key(e.namespace, e.url)))
+		binary.LittleEndian.PutUint64(n[:], uint64(len(e.data)))
+		h.Write(n[:])
+		h.Write(e.data)
+	}
+	var u [16]byte
+	copy(u[:], h.Sum(nil))
+	return u
+}

zim/zim_test.go

@@ -0,0 +1,136 @@
+package zim
+
+import (
+	"bytes"
+	"crypto/md5"
+	"strings"
+	"testing"
+)
+
+// buildSample writes a small archive exercising text, binary, metadata, a
+// redirect, and a main page, and returns its bytes.
+func buildSample(t *testing.T, noCompress bool) []byte {
+	t.Helper()
+	w := NewWriter()
+	w.SetNoCompress(noCompress)
+	w.AddContent(NamespaceContent, "index.html", "Home", "text/html",
+		[]byte("<h1>Home</h1>"+strings.Repeat(" word", 500)))
+	w.AddContent(NamespaceContent, "about/index.html", "About", "text/html",
+		[]byte("<h1>About</h1>"))
+	w.AddContent(NamespaceContent, "_kage/h/logo.png", "", "image/png",
+		[]byte{0x89, 'P', 'N', 'G', 0, 1, 2, 3, 4, 5})
+	w.AddMetadata("Title", "Sample")
+	w.AddMetadata("Language", "eng")
+	w.AddRedirect(NamespaceWellKnown, "mainPage", "Main", NamespaceContent, "index.html")
+	w.SetMainPage(NamespaceContent, "index.html")
+
+	var buf bytes.Buffer
+	n, err := w.WriteTo(&buf)
+	if err != nil {
+		t.Fatalf("WriteTo: %v", err)
+	}
+	if int(n) != buf.Len() {
+		t.Fatalf("WriteTo reported %d bytes, buffer has %d", n, buf.Len())
+	}
+	return buf.Bytes()
+}
+
+func TestRoundTrip(t *testing.T) {
+	for _, noCompress := range []bool{false, true} {
+		data := buildSample(t, noCompress)
+		r, err := NewReader(bytes.NewReader(data), int64(len(data)))
+		if err != nil {
+			t.Fatalf("NewReader (noCompress=%v): %v", noCompress, err)
+		}
+
+		// Content round-trips with the right mime.
+		home, err := r.Get(NamespaceContent, "index.html")
+		if err != nil {
+			t.Fatalf("Get home: %v", err)
+		}
+		if !strings.HasPrefix(string(home.Data), "<h1>Home</h1>") {
+			t.Errorf("home content wrong: %.20q", home.Data)
+		}
+		if home.MimeType != "text/html" {
+			t.Errorf("home mime = %q", home.MimeType)
+		}
+
+		// Binary blob survives byte-for-byte.
+		logo, err := r.Get(NamespaceContent, "_kage/h/logo.png")
+		if err != nil {
+			t.Fatalf("Get logo: %v", err)
+		}
+		if !bytes.Equal(logo.Data, []byte{0x89, 'P', 'N', 'G', 0, 1, 2, 3, 4, 5}) {
+			t.Errorf("logo bytes wrong: %v", logo.Data)
+		}
+		if logo.MimeType != "image/png" {
+			t.Errorf("logo mime = %q", logo.MimeType)
+		}
+
+		// Metadata.
+		meta, err := r.Get(NamespaceMetadata, "Title")
+		if err != nil || string(meta.Data) != "Sample" {
+			t.Errorf("metadata Title = %q, %v", meta.Data, err)
+		}
+
+		// Redirect resolves to the target's content.
+		red, err := r.Get(NamespaceWellKnown, "mainPage")
+		if err != nil {
+			t.Fatalf("Get redirect: %v", err)
+		}
+		if !strings.HasPrefix(string(red.Data), "<h1>Home</h1>") {
+			t.Errorf("redirect did not resolve to home: %.20q", red.Data)
+		}
+
+		// Main page.
+		mp, err := r.MainPage()
+		if err != nil {
+			t.Fatalf("MainPage: %v", err)
+		}
+		if !strings.HasPrefix(string(mp.Data), "<h1>Home</h1>") {
+			t.Errorf("main page wrong: %.20q", mp.Data)
+		}
+
+		// Misses error.
+		if _, err := r.Get(NamespaceContent, "nope.html"); err == nil {
+			t.Error("expected miss to error")
+		}
+	}
+}
+
+func TestChecksum(t *testing.T) {
+	data := buildSample(t, false)
+	if len(data) < 16 {
+		t.Fatal("archive too short")
+	}
+	body, sum := data[:len(data)-16], data[len(data)-16:]
+	want := md5.Sum(body)
+	if !bytes.Equal(sum, want[:]) {
+		t.Errorf("trailing MD5 does not match body hash")
+	}
+}
+
+func TestDeterministic(t *testing.T) {
+	a := buildSample(t, false)
+	b := buildSample(t, false)
+	if !bytes.Equal(a, b) {
+		t.Error("same input produced different archives; packing is not deterministic")
+	}
+}
+
+func TestMagicAndHeader(t *testing.T) {
+	data := buildSample(t, false)
+	h, err := parseHeader(data[:headerLen])
+	if err != nil {
+		t.Fatalf("parseHeader: %v", err)
+	}
+	if h.checksumPos != uint64(len(data)-16) {
+		t.Errorf("checksumPos = %d, want %d", h.checksumPos, len(data)-16)
+	}
+	if h.articleCount != 6 {
+		t.Errorf("articleCount = %d, want 6", h.articleCount)
+	}
+	if h.mainPage == noMainPage {
+		t.Error("main page not set in header")
+	}
+}