Drop the embedded leakless binary that trips antivirus (#69)

* Drop the embedded leakless binary that trips antivirus (#68)

go-rod's launcher imports github.com/ysmood/leakless, which base64/gzip
embeds a prebuilt leakless.exe for every target. On Windows that helper is
linked straight into kage.exe, and Windows Defender flags its signature and
quarantines a fresh scoop install before kage ever runs.

kage already launches Chrome with leakless disabled (browser/leakless.go),
so the guard was never doing anything, only adding the flagged bytes. This
adds an API-compatible stub for the package under third_party/leakless with
no embedded binary and points a replace directive at it. The Windows build
loses about 1.28 MB of packed executable and no longer carries the payload
that antivirus reacts to. Support() returns false, so go-rod skips the
leakless path even if a caller re-enabled it.

* Document the leakless antivirus fix for v0.3.9

Add the release-notes and changelog entries for the leakless helper removal
(#68), so the docs site and CHANGELOG explain why the Windows build shrank and
what the earlier virus warning was.

* Drop the now-unused leakless module hash from go.sum

The replace points leakless at a local directory, so go mod tidy no longer
needs the upstream module's checksum. Keeps the tidy CI check green.

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
@@ -34,7 +34,7 @@ jobs:
           cache: true
       - name: install chromium (linux)
         if: matrix.os == 'ubuntu-latest'
-        uses: browser-actions/setup-chrome@v1
+        uses: browser-actions/setup-chrome@v2.1.2
         id: chrome
       - name: gofmt
         run: |
@@ -67,13 +67,13 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
           check-latest: true
           cache: true
-      - uses: golangci/golangci-lint-action@v8
+      - uses: golangci/golangci-lint-action@v9.2.1
         with:
           version: latest
 
@@ -81,7 +81,7 @@ jobs:
   govulncheck:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
@@ -97,7 +97,7 @@ jobs:
   tidy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
@@ -117,7 +117,7 @@ jobs:
   webview:
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
@@ -133,7 +133,7 @@ jobs:
   windows-gui:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod

.github/workflows/docs.yml

@@ -26,14 +26,14 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6.0.2
+      - uses: actions/checkout@v7.0.0
         with:
           submodules: true
           # Sitemap lastmod comes from the latest content commit.
           fetch-depth: 0
 
       - name: Checkout tago
-        uses: actions/checkout@v6.0.2
+        uses: actions/checkout@v7.0.0
         with:
           repository: tamnd/tago
           path: .tago-src
@@ -107,7 +107,7 @@ jobs:
       group: cloudflare-pages-kage
       cancel-in-progress: true
     steps:
-      - uses: actions/checkout@v6.0.2
+      - uses: actions/checkout@v7.0.0
         with:
           fetch-depth: 1
           sparse-checkout: scripts/

.github/workflows/release.yml

@@ -27,15 +27,15 @@ jobs:
     if: github.ref_type != 'tag'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v6.4.0
         with:
           go-version-file: go.mod
           check-latest: true
           cache: true
-      - uses: goreleaser/goreleaser-action@v6
+      - uses: goreleaser/goreleaser-action@v7.2.2
         with:
           distribution: goreleaser
           version: "~> v2"
@@ -49,30 +49,38 @@ jobs:
       contents: write # create the GitHub release
       packages: write # push the image to ghcr.io
       id-token: write # keyless cosign signing
+    env:
+      LINUX_REPO_DISPATCH_TOKEN: ${{ secrets.LINUX_REPO_DISPATCH_TOKEN }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v6.4.0
         with:
           go-version-file: go.mod
           check-latest: true
           cache: true
 
       # Build and ship the linux/arm64 image from the amd64 runner.
-      - uses: docker/setup-qemu-action@v3
-      - uses: docker/setup-buildx-action@v3
-      - uses: docker/login-action@v3
+      - uses: docker/setup-qemu-action@v4.1.0
+      - uses: docker/setup-buildx-action@v4.1.0
+      - uses: docker/login-action@v4.2.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       # Tools GoReleaser shells out to for signing and SBOMs.
-      - uses: sigstore/cosign-installer@v3
-      - uses: anchore/sbom-action/download-syft@v0
+      # Pin cosign to the 2.x line. cosign 3.x makes the new bundle format the
+      # default, which ignores the --output-signature/--output-certificate flags
+      # the signs block uses and aborts. Pinning keeps the .sig/.pem outputs and
+      # stops the release tool from floating to a breaking latest.
+      - uses: sigstore/cosign-installer@v4.1.2
+        with:
+          cosign-release: "v2.6.3"
+      - uses: anchore/sbom-action/download-syft@v0.24.0
 
-      - uses: goreleaser/goreleaser-action@v6
+      - uses: goreleaser/goreleaser-action@v7.2.2
         with:
           distribution: goreleaser
           version: "~> v2"
@@ -85,3 +93,20 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }}
           SCOOP_BUCKET_GITHUB_TOKEN: ${{ secrets.SCOOP_BUCKET_GITHUB_TOKEN }}
+
+      # Rebuild the Linux apt/dnf repository with the packages just released.
+      # Skipped when the dispatch token is unset, so the release never depends
+      # on it being configured.
+      - name: Refresh the Linux package repository
+        if: env.LINUX_REPO_DISPATCH_TOKEN != ''
+        run: |
+          code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
+            -H "Authorization: Bearer $LINUX_REPO_DISPATCH_TOKEN" \
+            -H "Accept: application/vnd.github+json" \
+            https://api.github.com/repos/tamnd/linux-repo/dispatches \
+            -d '{"event_type":"package-released"}' || true)
+          if [ "$code" = "204" ]; then
+            echo "Linux repo refresh triggered"
+          else
+            echo "Linux repo refresh not accepted (HTTP $code); skipping"
+          fi

.goreleaser.yaml

@@ -151,14 +151,23 @@ homebrew_casks:
     repository:
       owner: tamnd
       name: homebrew-tap
-      token: '{{ envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}'
+      token: '{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}'
     directory: Casks
     homepage: https://github.com/tamnd/kage
     description: Clone any website for offline viewing, with the JavaScript stripped out
-    skip_upload: '{{ if envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}'
+    skip_upload: '{{ if index .Env "HOMEBREW_TAP_GITHUB_TOKEN" }}false{{ else }}true{{ end }}'
     commit_author:
       name: Duc-Tam Nguyen
       email: tamnd87@gmail.com
+    # Homebrew quarantines cask artifacts, and Gatekeeper kills quarantined
+    # binaries that are only ad-hoc signed (which cross-compiled Go binaries
+    # are). Strip the attribute at install so the binary runs first try.
+    hooks:
+      post:
+        install: |
+          if system_command("/usr/bin/xattr", args: ["-h"]).exit_status.zero?
+            system_command "/usr/bin/xattr", args: ["-dr", "com.apple.quarantine", "#{staged_path}/kage"]
+          end
 
 scoops:
   # Scoop manifest for Windows, pushed to the bucket repository. It installs the
@@ -168,11 +177,12 @@ scoops:
     repository:
       owner: tamnd
       name: scoop-bucket
-      token: '{{ envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}'
+      token: '{{ .Env.SCOOP_BUCKET_GITHUB_TOKEN }}'
+    directory: bucket
     homepage: https://github.com/tamnd/kage
     description: Clone any website for offline viewing, with the JavaScript stripped out
     license: MIT
-    skip_upload: '{{ if envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}'
+    skip_upload: '{{ if index .Env "SCOOP_BUCKET_GITHUB_TOKEN" }}false{{ else }}true{{ end }}'
     commit_author:
       name: Duc-Tam Nguyen
       email: tamnd87@gmail.com

CHANGELOG.md

@@ -6,6 +6,161 @@ All notable changes to kage are recorded here. The format follows
 
 ## [Unreleased]
 
+## [0.3.9] - 2026-07-08
+
+### Fixed
+
+- The Windows build no longer embeds the leakless watchdog binary that Windows Defender flags as `Trojan:Win32/Kepavll!rfn`, which made a fresh `scoop install` fail with a virus warning on `leakless.exe` ([#68](https://github.com/tamnd/kage/issues/68)).
+  go-rod's launcher imports [leakless](https://github.com/ysmood/leakless), which base64/gzip-embeds a prebuilt helper for every platform and links the Windows one into `kage.exe`.
+  kage already launches Chrome with leakless disabled, so the helper never ran, only added the flagged bytes.
+  A `replace` directive now points the package at an API-compatible stub under `third_party/leakless` that carries no embedded binary, dropping about 1.28 MB from the Windows build.
+
+## [0.3.6] - 2026-06-19
+
+### Fixed
+
+- `kage clone --mobile` now produces a correct layout when the source page uses table-based navigation with image maps, as paulgraham.com does.
+  Three rendering problems appeared in Kiwix iOS after the 0.3.5 release: a tall blank box at the top of every page, content clipped on the right edge, and a narrow spacer column occupying screen space.
+  The blank box came from hiding only the `<img usemap>` element while leaving its `<td>` container in place; the cell collapsed to empty but kept its full height.
+  `td:has(>img[usemap])` now hides the entire nav column rather than just the image inside it.
+  The right-side clip came from `width="435"` set directly as an HTML attribute on the inner content table; the earlier `max-width` CSS rule does not override HTML attributes.
+  A new `[width]{width:auto!important;max-width:100%!important}` rule cancels every fixed HTML width attribute on any element — tables, cells, and images alike — so the content column stretches to fill the phone screen.
+  The spacer column (a 26 px `<td>` holding a 1×1 transparent GIF) kept its allocated width for the same reason; `td:has(>img[src*="trans_1x1"]:only-child)` hides it alongside the nav column.
+  Two additional rules round out the fix: `*{box-sizing:border-box}` prevents padding from pushing content past the viewport edge, and `img{max-width:100%;height:auto}` keeps any inline photos within their column.
+
+## [0.3.5] - 2026-06-19
+
+### Changed
+
+- Each saved page is now stored in a packed ZIM under its own `<title>` instead of its URL path, so a ZIM reader's search box suggests pages by their readable title.
+  Typing into Kiwix's search now offers "Five Founders" or "Female Founders" rather than a filename, because the title pointer list a reader's suggestion search walks carries the real page titles.
+  A page with no `<title>` still falls back to its path, and the per-page title also flows into the `title` column of `kage parquet export`.
+
+### Added
+
+- `kage clone --mobile` makes legacy "font-era" sites readable on a phone.
+  Sites from the 1990s and early 2000s — paulgraham.com is a good example — embed typography directly in the HTML with `<font size="2" face="verdana">`, table-based layouts, and no viewport declaration.
+  A mobile browser receiving that markup without a viewport meta shrinks everything to desktop scale, and the `<font size="2">` instruction then makes the already-small text microscopic.
+  Passing `--mobile` injects two things into every saved page before it is written: a `<meta name="viewport" content="width=device-width, initial-scale=1">` tag so the browser stops shrinking, and a small `<style>` block that lifts the base font size to 18 px, inherits that size through `<font>` elements, caps the content width at 720 px, loosens line height to 1.7, and hides image-map navigation elements (usually a GIF served from an external CDN that 404s offline anyway).
+  The override is deliberately last in `<head>` so it wins specificity ties, and it does not touch pages that already carry a viewport and readable type sizes.
+- The packing guide now documents how search works on a kage archive: title suggestions in any ZIM reader, and full-text search of page bodies through `kage parquet export` and DuckDB.
+  A Xapian full-text index is deliberately not written, since Xapian is GPL and kage is MIT.
+
+## [0.3.4] - 2026-06-17
+
+### Fixed
+
+- `kage serve` now stops on Ctrl-C instead of ignoring it.
+  The preview server was started with a blocking call that never watched for an interrupt, so the only way to stop it was to kill the process.
+  kage now shuts the server down gracefully on an interrupt or a `SIGTERM`, with a short timeout before it forces the listener closed, so a preview exits cleanly. Thanks to Xirui Wang (#35) and Kaidi Zhao (#38).
+- A page whose JavaScript builds a deeply nested object graph no longer fails to clone.
+  Chrome's DevTools Protocol returns "Object reference chain is too long" while loading such a page, but the HTML has already loaded and the error is only about Chrome's internal object tracking, not the document.
+  kage now recognises that specific error and finishes rendering the page instead of dropping it (reported in #36). Thanks to Gautam Kumar (#39).
+
+## [0.3.3] - 2026-06-16
+
+### Fixed
+
+- Chrome no longer downloads a file to your Downloads folder when a crawl follows a link that turns out to be a binary (reported in #32).
+  An extensionless link is queued as a page, so the page worker navigated to it in Chrome, and a link that served a zip or a CSV made Chrome save the file to `~/Downloads`, a surprise side effect of a clone.
+  kage now denies Chrome-initiated downloads browser-wide, since every asset is fetched through kage's own downloader, and detects a navigation whose response is not HTML and reroutes that URL to the asset downloader, where the size and media policy decides whether to localise it or leave it on the live web.
+
+## [0.3.2] - 2026-06-16
+
+### Fixed
+
+- Saved pages now declare their character encoding, so text no longer mojibakes in a reader.
+  kage writes every page as UTF-8, but a source that set its charset only in the HTTP `Content-Type` header, with no `<meta charset>` in the markup, lost that signal once the page became a standalone file.
+  A reader serving the bytes without a charset then fell back to its locale encoding and turned every curly quote, dash, and non-breaking space into mojibake (reported in #16 and #29).
+  kage now inserts a `<meta charset="utf-8">` at the top of `<head>` when the page does not already declare one, so the page is self-describing in any reader.
+
+## [0.3.1] - 2026-06-15
+
+### Fixed
+
+- A served mirror whose entry point is a nested page no longer loses its CSS and
+  images when opened at the root. kage saves each page's asset links as
+  mirror-relative paths (`../_kage/...`) computed for that page's own location,
+  but the viewer answered `/` with the main page's bytes in place, so the browser
+  resolved those relative URLs against `/` and missed every one. A
+  `developer.apple.com/documentation` mirror, whose main page is
+  `developer.apple.com/documentation/index.html`, landed at `/` completely
+  unstyled. kage now redirects `/` to the main page's canonical content path, the
+  way the archive's `W/mainPage` redirect already does, so the browser resolves
+  the page's relative assets correctly. Kiwix was unaffected because it follows
+  that redirect itself.
+
+## [0.3.0] - 2026-06-15
+
+### Added
+
+- `kage parquet export <file.zim>` and `kage parquet import <file.parquet>`
+  convert a packed archive to a columnar Parquet table and back. The table is
+  flat, one row per archive entry, with clear columns (url, host, title, mime,
+  extracted text, content), so it drops straight into the tooling a dataset host
+  like Hugging Face expects, and DuckDB or pandas can query it as is. The column
+  names follow the open-index/open-markdown dataset (`doc_id`, `url`, `host`,
+  `crawl_date`, `content_length`, `text_length`, `text`), with `doc_id` a
+  deterministic UUID v5 of the page URL, so a kage export sits alongside other
+  web-crawl datasets. The conversion is lossless: a ZIM round-tripped through
+  Parquet reproduces every entry, its metadata, and the main page byte for byte.
+- `kage pack --incremental` keeps a small cache sidecar next to the output and
+  reuses the compression of any cluster whose bytes have not changed since the
+  last pack. Compressing clusters with zstd is the dominant cost of packing a
+  large mirror, so re-packing after a small change (a `--refresh`, a handful of
+  edited pages) only compresses what actually changed instead of the whole
+  archive. A cached cluster is byte-for-byte what a fresh compression produces,
+  so the archive stays deterministic and valid. The pack reports how many
+  clusters it reused versus compressed.
+- Identical pages are now stored once. When a rendered page's bytes match a page
+  already written, kage stores it as a hard link to the first copy instead of a
+  second full file. This collapses the duplicate content a faceted site spawns
+  when many `?q=…`/`?page=…` URLs all render the same page. The final summary
+  reports how many pages were deduped this way.
+
+### Changed
+
+- Clones no longer pull a site's bulk downloads into the mirror by default. Video
+  and audio, installers and disk images (`.dmg`, `.pkg`, `.exe`, `.msi`, ...),
+  archives, and PDFs are left pointing at their live URL instead of downloaded,
+  because they are rarely needed to read a site offline yet routinely make up
+  most of its bytes (a developer.apple.com crawl was 18 of 19 GB of such assets).
+  Page-rendering assets (images, fonts, CSS) are unaffected. `--keep-media`
+  restores the old behavior, and `--skip-ext .foo` leaves more extensions remote.
+- Assets are localized only from the seed's own registrable domain by default.
+  developer.apple.com still pulls from www.apple.com and images.apple.com, but a
+  separate brand domain or an unrelated third party (an embedded tracker, an
+  off-topic CDN) is left on the live web rather than mirrored. `--all-asset-hosts`
+  restores downloading assets from any host.
+- The progress line now counts real pages. "pages" is the number of distinct URL
+  paths written, and the query-string variants that one path can spawn by the
+  thousand are shown separately as "variants", so the live counter tracks the
+  site's real size instead of being inflated by `?q=…` permutations.
+
+### Fixed
+
+- An asset larger than the size cap (`--max-asset-mb`, 25 by default) is now
+  skipped instead of being truncated to a corrupt fragment. The cap was a
+  `LimitReader`, so an over-size file was saved as exactly the first N MB of
+  itself: a broken video or installer that wasted disk and would never play or
+  run. kage now checks the response size and leaves an over-cap asset out of the
+  mirror entirely, pointing at its live URL. On a developer.apple.com crawl this
+  was around a gigabyte of truncated WWDC videos and `.dmg` installers.
+- An asset URL whose query string carries a raw space is now requested with the
+  space percent-encoded, so the server gets a valid request instead of rejecting
+  it. Real sites bust a stylesheet's cache with a date, producing an href like
+  `styles/main.css?Thursday, 26-Feb-2026 16:26:41 UTC`; a browser encodes the
+  spaces before requesting, but kage was passing them through verbatim and the
+  server answered `400 Bad Request`. On a developer.apple.com crawl this was the
+  cause of the large majority of the download errors. The query is re-encoded on
+  the canonical URL, so the on-disk key matches the fixed request.
+
+## [0.2.1] - 2026-06-15
+
+### Added
+
+- ZIM archives now carry the metadata Kiwix and `zimcheck` treat as mandatory. Every archive gets a `Name` and a `Description` (a host-derived line when `--description` is not given), and, when the mirror has a usable favicon, an `Illustrator_48x48@1` entry: the icon rescaled to a 48x48 PNG, which is the book icon Kiwix shows in its library.
+
 ## [0.2.0] - 2026-06-15
 
 ### Added
@@ -109,7 +264,14 @@ can browse offline, with every script stripped out.
   a multi-arch container image on GHCR (Chromium bundled), checksums, SBOMs, and
   a cosign signature, all cut from one version tag by GoReleaser.
 
-[Unreleased]: https://github.com/tamnd/kage/compare/v0.2.0...HEAD
+[Unreleased]: https://github.com/tamnd/kage/compare/v0.3.9...HEAD
+[0.3.9]: https://github.com/tamnd/kage/compare/v0.3.8...v0.3.9
+[0.3.4]: https://github.com/tamnd/kage/compare/v0.3.3...v0.3.4
+[0.3.3]: https://github.com/tamnd/kage/compare/v0.3.2...v0.3.3
+[0.3.2]: https://github.com/tamnd/kage/compare/v0.3.1...v0.3.2
+[0.3.1]: https://github.com/tamnd/kage/compare/v0.3.0...v0.3.1
+[0.3.0]: https://github.com/tamnd/kage/compare/v0.2.1...v0.3.0
+[0.2.1]: https://github.com/tamnd/kage/compare/v0.2.0...v0.2.1
 [0.2.0]: https://github.com/tamnd/kage/compare/v0.1.2...v0.2.0
 [0.1.2]: https://github.com/tamnd/kage/compare/v0.1.1...v0.1.2
 [0.1.1]: https://github.com/tamnd/kage/compare/v0.1.0...v0.1.1

Dockerfile

@@ -17,24 +17,28 @@ ARG TARGETPLATFORM
 # chromium for rendering; ca-certificates for HTTPS; tzdata for sane timestamps;
 # the font package so rendered pages have glyphs to lay out.
 RUN apk add --no-cache chromium ca-certificates tzdata font-noto \
- && adduser -D -H -u 10001 kage \
- && mkdir -p /out \
- && chown kage:kage /out
+ && mkdir -p /out
 
 COPY $TARGETPLATFORM/kage /usr/bin/kage
 
-USER kage
 WORKDIR /out
 
 # Point kage at the bundled Chromium and write mirrors under /out by default:
 #
 #   docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com
 #
-# The kage user has no home directory of its own, so HOME points at the mounted
-# /out volume. That keeps two things writable: kage's default output and resume
-# state (it lands under $HOME/data/kage), and Chrome's profile and crash
-# database. Without this both fail with a permission error in the container
-# (issue #7), and the mounted volume captures nothing.
+# The container runs as root, and that is deliberate (issue #7). A bind-mounted
+# /out is owned by whoever created it on the host, so only root can reliably
+# write into it; a fixed non-root uid cannot, and both kage's output and resume
+# state (under $HOME/data/kage) then fail with "mkdir /out: permission denied".
+# The same unwritable HOME also breaks Chrome: it launches chrome_crashpad_handler
+# with an empty crash database path, which aborts the whole browser with
+# "chrome_crashpad_handler: --database is required" and fails every render.
+# Running as root keeps /out and HOME writable whatever the host owns, so the
+# one-liner above just works. This costs nothing in the sandbox: Chrome's sandbox
+# is already off inside any container (kage drops it on container detection), so
+# root here does not loosen a boundary that was holding. HOME points at /out so
+# the default output and Chrome's writable state both land in the mounted volume.
 ENV KAGE_CHROME=/usr/bin/chromium-browser \
     HOME=/out
 

README.md

@@ -24,7 +24,27 @@ Full docs and guides live at **[kage.tamnd.com](https://kage.tamnd.com)**.
 go install github.com/tamnd/kage/cmd/kage@latest
 ```
 
-Prefer a prebuilt binary? Grab an archive, a `.deb`/`.rpm`/`.apk`, or a checksum from [releases](https://github.com/tamnd/kage/releases). Or skip installing Chrome yourself and use the container image, which bundles Chromium:
+Prefer a prebuilt binary? Grab an archive, a `.deb`/`.rpm`/`.apk`, or a checksum from [releases](https://github.com/tamnd/kage/releases). Or let a package manager handle it:
+
+```bash
+# Homebrew (macOS)
+brew install --cask tamnd/tap/kage
+
+# Scoop (Windows)
+scoop bucket add tamnd https://github.com/tamnd/scoop-bucket
+scoop install kage
+
+# apt (Debian, Ubuntu)
+curl -fsSL https://tamnd.github.io/linux-repo/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg
+echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" | sudo tee /etc/apt/sources.list.d/tamnd.list
+sudo apt update && sudo apt install kage
+
+# dnf (Fedora, RHEL)
+sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo
+sudo dnf install kage
+```
+
+Or skip installing Chrome yourself and use the container image, which bundles Chromium:
 
 ```bash
 docker run --rm -v "$PWD/out:/out" ghcr.io/tamnd/kage clone paulgraham.com
@@ -102,6 +122,7 @@ The flags you'll actually reach for:
 | `--scroll` | `false` | Auto-scroll each page to trigger lazy loading |
 | `--workers` | `4` | How many pages to render at once |
 | `--no-robots` | `false` | Ignore `robots.txt` (be nice) |
+| `--crawl-delay` | `0s` | Override robots.txt `Crawl-delay` between page starts |
 | `-f, --force` | `false` | Delete any existing mirror for the host first |
 | `--chrome` | | Path to the Chrome/Chromium binary |
 

asset/asset_test.go

@@ -69,6 +69,7 @@ body { background: url(../img/bg.png); }
 
 const htmlDoc = `<!doctype html><html><head>
 <link rel="stylesheet" href="/css/main.css">
+<link rel="preload stylesheet" href="/css/vp.css" as="style">
 <link rel="icon" href="/favicon.ico">
 <link rel="canonical" href="https://ex.com/canon">
 </head><body>
@@ -98,6 +99,7 @@ func TestRewriteHTML(t *testing.T) {
 
 	checks := map[string]bool{
 		`href="_kage/ex.com/css/main.css"`:                  true, // stylesheet localised
+		`href="_kage/ex.com/css/vp.css"`:                    true, // multi-value "preload stylesheet" rel localised
 		`href="_kage/ex.com/favicon.ico"`:                   true, // icon localised
 		`href="https://ex.com/canon"`:                       true, // canonical left alone
 		`href="docs/intro/index.html"`:                      true, // internal page → local

asset/download.go

@@ -42,6 +42,13 @@ type Result struct {
 	IsCSS       bool
 }
 
+// ErrTooLarge reports that an asset exceeds the size cap and was skipped without
+// being saved. It is deliberately a skip, not a download failure: the caller
+// leaves the asset out of the mirror rather than writing a truncated fragment of
+// it, so a 500 MB installer or video never bloats the archive with a corrupt
+// quarter of itself.
+var ErrTooLarge = errors.New("asset over size cap")
+
 // StatusError reports a non-2xx HTTP response. It carries the code so callers
 // can render a clear message ("HTTP 403 Forbidden") and decide whether a retry
 // is worthwhile, without the URL baked in (the caller already has it).
@@ -109,14 +116,26 @@ func (d *Downloader) try(ctx context.Context, u *url.URL, referer string) (*Resu
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		return nil, &StatusError{Code: resp.StatusCode}
 	}
+	// Skip an over-cap asset instead of truncating it. A Content-Length lets us
+	// bail before reading a byte; otherwise we read one byte past the cap and, if
+	// the body really is larger, discard what we have. Either way nothing partial
+	// reaches disk.
+	if d.MaxBytes > 0 && resp.ContentLength > d.MaxBytes {
+		return nil, ErrTooLarge
+	}
 	var r io.Reader = resp.Body
 	if d.MaxBytes > 0 {
-		r = io.LimitReader(resp.Body, d.MaxBytes)
+		// Read at most one byte past the cap so a body with no (or a lying)
+		// Content-Length cannot stream gigabytes into memory before we notice.
+		r = io.LimitReader(resp.Body, d.MaxBytes+1)
 	}
 	body, err := io.ReadAll(r)
 	if err != nil {
 		return nil, err
 	}
+	if d.MaxBytes > 0 && int64(len(body)) > d.MaxBytes {
+		return nil, ErrTooLarge
+	}
 	ct := resp.Header.Get("Content-Type")
 	return &Result{
 		Body:        body,
@@ -142,6 +161,9 @@ func transient(err error) bool {
 	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
 		return false
 	}
+	if errors.Is(err, ErrTooLarge) {
+		return false
+	}
 	var se *StatusError
 	if errors.As(err, &se) {
 		switch se.Code {

asset/download_test.go

@@ -94,6 +94,69 @@ func TestGetGivesUpAfterRetries(t *testing.T) {
 	}
 }
 
+func TestGetSkipsOverCapByContentLength(t *testing.T) {
+	var hits int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hits, 1)
+		body := make([]byte, 4096) // declared via Content-Length
+		w.Header().Set("Content-Type", "video/mp4")
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	d := NewDownloader("kage-test", 5*time.Second, 1024) // cap below the body
+	u, _ := url.Parse(srv.URL + "/clip.mp4")
+	_, err := d.Get(context.Background(), u, "")
+	if !errors.Is(err, ErrTooLarge) {
+		t.Fatalf("got %v; want ErrTooLarge", err)
+	}
+	if hits != 1 {
+		t.Errorf("an over-cap asset should not be retried; server saw %d hits", hits)
+	}
+}
+
+func TestGetSkipsOverCapWithoutContentLength(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// A chunked response carries no Content-Length, so the cap is only known
+		// after reading past it.
+		w.Header().Set("Content-Type", "application/octet-stream")
+		fl, _ := w.(http.Flusher)
+		chunk := make([]byte, 512)
+		for i := 0; i < 8; i++ {
+			_, _ = w.Write(chunk)
+			if fl != nil {
+				fl.Flush()
+			}
+		}
+	}))
+	defer srv.Close()
+
+	d := NewDownloader("kage-test", 5*time.Second, 1024)
+	u, _ := url.Parse(srv.URL + "/stream.bin")
+	_, err := d.Get(context.Background(), u, "")
+	if !errors.Is(err, ErrTooLarge) {
+		t.Fatalf("got %v; want ErrTooLarge", err)
+	}
+}
+
+func TestGetKeepsUnderCap(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "image/png")
+		_, _ = w.Write([]byte("small"))
+	}))
+	defer srv.Close()
+
+	d := NewDownloader("kage-test", 5*time.Second, 1024)
+	u, _ := url.Parse(srv.URL + "/logo.png")
+	res, err := d.Get(context.Background(), u, "")
+	if err != nil {
+		t.Fatalf("under-cap asset: %v", err)
+	}
+	if string(res.Body) != "small" {
+		t.Errorf("body = %q; want %q", res.Body, "small")
+	}
+}
+
 func TestTransientClassification(t *testing.T) {
 	transientCodes := []int{403, 408, 425, 429, 500, 502, 503}
 	for _, c := range transientCodes {

asset/html.go

@@ -9,13 +9,28 @@ import (
 	"golang.org/x/net/html/atom"
 )
 
-// stylesheetRels are <link rel> values whose href kage downloads as an asset.
-var stylesheetRels = map[string]bool{
-	"stylesheet": true, "icon": true, "shortcut icon": true,
+// assetRels are the individual <link rel> tokens whose href kage downloads as
+// an asset. A rel attribute is a space-separated token list, so a single known
+// token in it (for example the "stylesheet" in "preload stylesheet") is enough.
+var assetRels = map[string]bool{
+	"stylesheet": true, "icon": true,
 	"apple-touch-icon": true, "apple-touch-icon-precomposed": true,
 	"mask-icon": true, "manifest": true, "preload": true, "prefetch": true,
 }
 
+// linkRelDownloadable reports whether a <link rel> names a resource kage should
+// download. It treats rel as the space-separated token list the HTML spec
+// defines, so "preload stylesheet", "shortcut icon", or a bare "stylesheet" all
+// match on a single recognised token.
+func linkRelDownloadable(rel string) bool {
+	for _, tok := range strings.Fields(strings.ToLower(rel)) {
+		if assetRels[tok] {
+			return true
+		}
+	}
+	return false
+}
+
 // RewriteHTML walks the parsed document and rewrites every resource and link
 // reference through sink, resolving relative URLs against base. It mutates the
 // tree in place; the caller renders it afterwards. References kage cannot handle
@@ -40,8 +55,7 @@ func rewriteElement(n *html.Node, base *url.URL, sink RefSink) {
 	case atom.Iframe, atom.Frame:
 		rewriteAttr(n, "src", base, sink, pageOrAsset)
 	case atom.Link:
-		rel := strings.ToLower(strings.TrimSpace(attrVal(n, "rel")))
-		if stylesheetRels[rel] {
+		if linkRelDownloadable(attrVal(n, "rel")) {
 			rewriteAttr(n, "href", base, sink, alwaysAsset)
 		}
 	case atom.Img:

browser/leakless.go

@@ -0,0 +1,7 @@
+package browser
+
+import "runtime"
+
+func launcherLeakless() bool {
+	return runtime.GOOS != "windows"
+}

browser/pool.go

@@ -67,6 +67,21 @@ type RenderResult struct {
 	Title    string
 }
 
+// ErrNotHTML reports that a URL kage tried to render as a page is not HTML: the
+// server returned some other content type (a zip, a CSV, a PDF, a bare image).
+// Such a URL reaches the page worker when its link carried no file extension to
+// classify it by. The caller reroutes it to the asset downloader, where the
+// asset policy decides whether to localise or leave it remote, instead of saving
+// an empty or broken page or letting Chrome download it (issue #32).
+type ErrNotHTML struct {
+	URL         string
+	ContentType string
+}
+
+func (e *ErrNotHTML) Error() string {
+	return fmt.Sprintf("not HTML (%s): %s", e.ContentType, e.URL)
+}
+
 // Render navigates to rawURL, lets it settle, and returns the final rendered
 // HTML. It acquires a page slot from the pool and releases it when done.
 func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error) {
@@ -90,11 +105,33 @@ func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error)
 
 	page = page.Context(ctx).Timeout(p.opts.RenderTimeout)
 
-	if err := page.Navigate(rawURL); err != nil {
-		return RenderResult{}, fmt.Errorf("navigate %s: %w", rawURL, err)
+	// Watch the main document's response so a navigation that turns out to be a
+	// non-HTML resource (a zip, a CSV, a bare image) is caught and handed back for
+	// the asset downloader, rather than rendered as a broken page or, with downloads
+	// denied, left as an aborted navigation (issue #32). The content type arrives in
+	// the response headers whether Chrome renders the body or aborts it as a denied
+	// download, so this catches both.
+	mainContentType := watchMainDocument(page)
+
+	navErr := page.Navigate(rawURL)
+	// A denied download aborts the navigation, so inspect the captured content type
+	// before treating a navigation error as a failure. waitFor gives the response
+	// event a brief moment to be processed; for an HTML page it returns at once.
+	if ct := waitFor(ctx, mainContentType, 2*time.Second); ct != "" && !isHTML(ct) {
+		return RenderResult{}, &ErrNotHTML{URL: rawURL, ContentType: ct}
+	}
+	if navErr != nil {
+		return RenderResult{}, fmt.Errorf("navigate %s: %w", rawURL, navErr)
 	}
 	if err := page.WaitLoad(); err != nil {
-		return RenderResult{}, fmt.Errorf("wait load %s: %w", rawURL, err)
+		// Chrome's DevTools Protocol may return "Object reference chain is too
+		// long" when a page's JavaScript builds deeply nested object graphs.
+		// The page has still loaded its HTML — the error is only about Chrome's
+		// internal object tracking, not about the document. Log the warning and
+		// continue rendering rather than failing the entire page (issue #36).
+		if !isObjRefChainError(err) {
+			return RenderResult{}, fmt.Errorf("wait load %s: %w", rawURL, err)
+		}
 	}
 	settle(page, p.opts.Settle)
 	if p.opts.Scroll {
@@ -128,7 +165,7 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
 
 	controlURL := p.opts.ControlURL
 	if controlURL == "" {
-		l := launcher.New().
+		l := launcher.New().Leakless(launcherLeakless()).
 			Headless(p.opts.Headless).
 			Set("disable-blink-features", "AutomationControlled").
 			Set("disable-gpu", "")
@@ -146,13 +183,14 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
 		// In a container, the default /dev/shm is only 64 MB, too small for
 		// Chrome's renderer on large pages, so steer it to a temp file instead.
 		// Outside a container /dev/shm is roomy and faster, so leave it alone.
-		// Chrome's crashpad handler also aborts with "--database is required" in a
-		// minimal container, which fails the whole launch (issue #7), so turn the
-		// crash reporter off there. kage never uploads Chrome crash dumps anyway.
+		//
+		// The "chrome_crashpad_handler: --database is required" abort seen in
+		// containers (issue #7) is not fixed here: the crash-reporter flags do not
+		// stop Chrome from spawning the handler. Its real cause is an unwritable
+		// HOME, which leaves the crash database path empty; the image keeps HOME
+		// writable instead (see the Dockerfile).
 		if inContainer() {
-			l = l.Set("disable-dev-shm-usage", "").
-				Set("disable-crash-reporter", "").
-				Set("disable-breakpad", "")
+			l = l.Set("disable-dev-shm-usage", "")
 		}
 
 		if bin := p.chromeBin(); bin != "" {
@@ -169,6 +207,21 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
 	if err := b.Connect(); err != nil {
 		return nil, fmt.Errorf("connect Chrome: %w", err)
 	}
+
+	// kage never wants Chrome to write a file to disk. Every asset is fetched
+	// through kage's own downloader, which applies the size and media policy, so a
+	// Chrome-initiated download is only ever an accident: navigating an <a> link
+	// that turns out to be a binary (a zip, an installer, a CSV) makes Chrome save
+	// it to the user's Downloads folder, a surprise side effect of a crawl
+	// (issue #32). Denying downloads browser-wide stops that. The navigation is
+	// aborted instead, and Render's non-HTML detection reroutes the URL through the
+	// asset downloader, where the asset policy decides its fate. This is
+	// best-effort: if the call is unsupported, the non-HTML detection still keeps
+	// the binary out of the saved mirror.
+	_ = proto.BrowserSetDownloadBehavior{
+		Behavior: proto.BrowserSetDownloadBehaviorBehaviorDeny,
+	}.Call(b)
+
 	p.browser = b
 	return b, nil
 }
@@ -321,6 +374,83 @@ func envBool(name string) (val, ok bool) {
 	}
 }
 
+// watchMainDocument subscribes to network responses and returns an accessor for
+// the main document's content type. The first Document-type response is the main
+// frame's navigation; later Document responses are sub-frames (iframes), whose
+// type kage does not police, so only the first is kept. The accessor is safe to
+// call from another goroutine. Any setup error leaves the accessor returning "",
+// which the caller reads as "unknown, render normally".
+func watchMainDocument(page *rod.Page) func() string {
+	var (
+		mu sync.Mutex
+		ct string
+	)
+	if err := (proto.NetworkEnable{}).Call(page); err != nil {
+		return func() string { return "" }
+	}
+	wait := page.EachEvent(func(e *proto.NetworkResponseReceived) {
+		if e.Type != proto.NetworkResourceTypeDocument || e.Response == nil {
+			return
+		}
+		mu.Lock()
+		if ct == "" {
+			ct = e.Response.MIMEType
+		}
+		mu.Unlock()
+	})
+	// EachEvent's wait blocks until the page context ends, draining events as they
+	// arrive; run it for the page's lifetime. The deferred page.Close in Render
+	// cancels the context and unblocks it.
+	go wait()
+	return func() string {
+		mu.Lock()
+		defer mu.Unlock()
+		return ct
+	}
+}
+
+// waitFor polls get until it returns a non-empty value, the deadline passes, or
+// the context is cancelled, then returns whatever it last saw. It exists because
+// the network response is processed on another goroutine, so the value may not be
+// set the instant Navigate returns; an HTML page sets it within a few
+// milliseconds, while a never-arriving response simply waits out the deadline.
+func waitFor(ctx context.Context, get func() string, deadline time.Duration) string {
+	const step = 20 * time.Millisecond
+	for waited := time.Duration(0); waited < deadline; waited += step {
+		if v := get(); v != "" {
+			return v
+		}
+		select {
+		case <-ctx.Done():
+			return get()
+		case <-time.After(step):
+		}
+	}
+	return get()
+}
+
+// isHTML reports whether a document content type is one kage renders and saves as
+// a page. HTML and XHTML qualify; an empty type is treated as HTML so an unlabelled
+// response still renders. Anything else (a zip, a CSV, a PDF, a bare image or
+// JSON) is an asset that reached the page worker because its link carried no file
+// extension to classify it by.
+func isHTML(contentType string) bool {
+	mt := strings.ToLower(strings.TrimSpace(contentType))
+	if i := strings.IndexByte(mt, ';'); i >= 0 {
+		mt = strings.TrimSpace(mt[:i])
+	}
+	return mt == "" || mt == "text/html" || mt == "application/xhtml+xml"
+}
+
+// isObjRefChainError reports whether err is the Chrome DevTools Protocol error
+// "Object reference chain is too long" (code -32000). This surfaces when a
+// page's JavaScript builds deeply nested object graphs. The page has still
+// loaded — Chrome's internal state tracking hit a limit, not the document
+// itself (issue #36).
+func isObjRefChainError(err error) bool {
+	return err != nil && strings.Contains(err.Error(), "Object reference chain is too long")
+}
+
 // settle waits for the network to go quiet for d, recovering from any rod
 // panic and capping the wait so a chatty page can never hang the worker.
 func settle(page *rod.Page, d time.Duration) {

browser/pool_test.go

@@ -2,9 +2,11 @@ package browser
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -82,6 +84,14 @@ func TestDisableSandboxContainer(t *testing.T) {
 	}
 }
 
+func TestLauncherLeaklessDisabledOnWindows(t *testing.T) {
+	got := launcherLeakless()
+	want := runtime.GOOS != "windows"
+	if got != want {
+		t.Errorf("launcherLeakless() = %v on %s; want %v", got, runtime.GOOS, want)
+	}
+}
+
 func TestRenderCapturesFinalDOM(t *testing.T) {
 	if testing.Short() {
 		t.Skip("render test drives Chrome; skipped under -short")
@@ -115,3 +125,86 @@ func TestRenderCapturesFinalDOM(t *testing.T) {
 		t.Errorf("render did not capture the JS-built DOM:\n%s", res.HTML)
 	}
 }
+
+func TestIsHTML(t *testing.T) {
+	cases := []struct {
+		ct   string
+		want bool
+	}{
+		{"text/html", true},
+		{"text/html; charset=utf-8", true},
+		{"TEXT/HTML", true},
+		{"  text/html ", true},
+		{"application/xhtml+xml", true},
+		{"", true}, // unknown: render rather than misclassify
+		{"application/zip", false},
+		{"text/csv", false},
+		{"application/pdf", false},
+		{"image/png", false},
+		{"application/json", false},
+		{"application/octet-stream", false},
+	}
+	for _, c := range cases {
+		if got := isHTML(c.ct); got != c.want {
+			t.Errorf("isHTML(%q) = %v, want %v", c.ct, got, c.want)
+		}
+	}
+}
+
+func TestRenderRoutesNonHTML(t *testing.T) {
+	if testing.Short() {
+		t.Skip("render test drives Chrome; skipped under -short")
+	}
+	if _, ok := LookChrome(); !ok {
+		t.Skip("no Chrome/Chromium found; skipping render test")
+	}
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/page":
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			_, _ = w.Write([]byte(`<!doctype html><html><body><p>a real page</p></body></html>`))
+		case "/file.zip", "/download":
+			// A binary served with no useful extension on the path, the shape that
+			// makes Chrome download to ~/Downloads when navigated to (issue #32).
+			w.Header().Set("Content-Type", "application/zip")
+			_, _ = w.Write([]byte("PK\x03\x04 not really a zip"))
+		case "/data":
+			w.Header().Set("Content-Type", "text/csv")
+			_, _ = w.Write([]byte("a,b\n1,2\n"))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	p := New(Options{Headless: true, Workers: 1, Settle: 300 * time.Millisecond, RenderTimeout: 20 * time.Second})
+	defer func() { _ = p.Close() }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// A real HTML page renders as before.
+	if res, err := p.Render(ctx, srv.URL+"/page"); err != nil {
+		t.Errorf("render HTML page: %v", err)
+	} else if !strings.Contains(res.HTML, "a real page") {
+		t.Errorf("HTML page did not render:\n%s", res.HTML)
+	}
+
+	// Non-HTML navigation targets come back as *ErrNotHTML so the caller can route
+	// them to the asset downloader instead of saving a broken page or downloading.
+	for _, tc := range []struct{ path, wantCT string }{
+		{"/download", "application/zip"},
+		{"/data", "text/csv"},
+	} {
+		_, err := p.Render(ctx, srv.URL+tc.path)
+		var notHTML *ErrNotHTML
+		if !errors.As(err, &notHTML) {
+			t.Errorf("Render(%s) error = %v, want *ErrNotHTML", tc.path, err)
+			continue
+		}
+		if !strings.Contains(notHTML.ContentType, tc.wantCT) {
+			t.Errorf("Render(%s) content type = %q, want %q", tc.path, notHTML.ContentType, tc.wantCT)
+		}
+	}
+}

cli/clone.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -15,33 +16,38 @@ import (
 
 // cloneFlags holds the parsed flag values for one invocation.
 type cloneFlags struct {
-	out          string
-	reserved     string
-	workers      int
-	assetWorkers int
-	browserPages int
-	maxPages     int
-	maxDepth     int
-	traversal    string
-	maxAssetMB   int64
-	timeout      time.Duration
-	settle       time.Duration
-	renderTO     time.Duration
-	scroll       bool
-	userAgent    string
-	subdomains   bool
-	scopePrefix  string
-	exclude      []string
-	noRobots     bool
-	noSitemap    bool
-	headful      bool
-	keepNoscript bool
-	chromeBin    string
-	controlURL   string
-	noResume     bool
-	refresh      bool
-	force        bool
-	quiet        bool
+	out            string
+	reserved       string
+	workers        int
+	assetWorkers   int
+	browserPages   int
+	maxPages       int
+	maxDepth       int
+	traversal      string
+	maxAssetMB     int64
+	keepMedia      bool
+	skipExt        []string
+	allAssetHosts  bool
+	timeout        time.Duration
+	settle         time.Duration
+	renderTO       time.Duration
+	scroll         bool
+	userAgent      string
+	subdomains     bool
+	scopePrefix    string
+	exclude        []string
+	noRobots       bool
+	crawlDelay     time.Duration
+	noSitemap      bool
+	headful        bool
+	keepNoscript   bool
+	mobileReadable bool
+	chromeBin      string
+	controlURL     string
+	noResume       bool
+	refresh        bool
+	force          bool
+	quiet          bool
 }
 
 func newCloneCmd() *cobra.Command {
@@ -66,7 +72,10 @@ func newCloneCmd() *cobra.Command {
 	fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "stop after N pages (0 = unlimited)")
 	fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)")
 	fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order: bfs or dfs")
-	fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB")
+	fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB (left on the live web)")
+	fs.BoolVar(&f.keepMedia, "keep-media", false, "download bulk media, installers, and PDFs instead of leaving them remote")
+	fs.StringSliceVar(&f.skipExt, "skip-ext", nil, "extra asset extensions to leave remote, e.g. .svg (repeatable)")
+	fs.BoolVar(&f.allAssetHosts, "all-asset-hosts", false, "localize assets from any host, not just the seed's domain")
 	fs.DurationVar(&f.timeout, "timeout", 30*time.Second, "per-request timeout")
 	fs.DurationVar(&f.settle, "settle", 1500*time.Millisecond, "network-idle quiet period before snapshot")
 	fs.DurationVar(&f.renderTO, "render-timeout", 30*time.Second, "hard cap per page render")
@@ -76,9 +85,11 @@ func newCloneCmd() *cobra.Command {
 	fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix")
 	fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)")
 	fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)")
+	fs.DurationVar(&f.crawlDelay, "crawl-delay", 0, "override robots.txt Crawl-delay between page starts (0 = use robots.txt)")
 	fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml")
 	fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)")
 	fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap <noscript> content instead of dropping it")
+	fs.BoolVar(&f.mobileReadable, "mobile", false, "inject viewport and CSS overrides so legacy sites read comfortably on a phone")
 	fs.StringVar(&f.chromeBin, "chrome", "", "path to the Chrome/Chromium binary")
 	fs.StringVar(&f.controlURL, "control-url", "", "attach to an existing Chrome DevTools endpoint")
 	fs.BoolVar(&f.noResume, "no-resume", false, "do not reuse or write resume state")
@@ -93,6 +104,9 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
 	if err != nil {
 		return fmt.Errorf("invalid url %q: %w", arg, err)
 	}
+	if f.crawlDelay < 0 {
+		return fmt.Errorf("--crawl-delay must be >= 0")
+	}
 
 	cfg := clone.DefaultConfig()
 	cfg.OutDir = f.out
@@ -104,6 +118,23 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
 	cfg.MaxDepth = f.maxDepth
 	cfg.Traversal = f.traversal
 	cfg.MaxAssetBytes = f.maxAssetMB << 20
+	cfg.AssetSameDomain = !f.allAssetHosts
+	if f.keepMedia {
+		cfg.SkipAssetExts = map[string]bool{}
+	}
+	for _, e := range f.skipExt {
+		e = strings.ToLower(strings.TrimSpace(e))
+		if e == "" {
+			continue
+		}
+		if !strings.HasPrefix(e, ".") {
+			e = "." + e
+		}
+		if cfg.SkipAssetExts == nil {
+			cfg.SkipAssetExts = map[string]bool{}
+		}
+		cfg.SkipAssetExts[e] = true
+	}
 	cfg.Timeout = f.timeout
 	cfg.Settle = f.settle
 	cfg.RenderTimeout = f.renderTO
@@ -113,9 +144,11 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
 	cfg.ScopePrefix = f.scopePrefix
 	cfg.ExcludePaths = f.exclude
 	cfg.RespectRobots = !f.noRobots
+	cfg.CrawlDelay = f.crawlDelay
 	cfg.FollowSitemap = !f.noSitemap
 	cfg.Headless = !f.headful
 	cfg.KeepNoscript = f.keepNoscript
+	cfg.MobileReadable = f.mobileReadable
 	cfg.ChromeBin = f.chromeBin
 	cfg.ControlURL = f.controlURL
 	cfg.Resume = !f.noResume
@@ -172,18 +205,33 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
 	return nil
 }
 
-// progressLine renders the single-line live counter.
+// progressLine renders the single-line live counter. "pages" is the count of
+// real pages (distinct paths); when a faceted site spawns query-string variants
+// they are shown separately so the page number stays easy to read.
 func progressLine(p clone.Progress) string {
+	if variants := p.Pages - p.PagePaths; variants > 0 {
+		return styleDim.Render(fmt.Sprintf("pages %d  variants %d  assets %d  errors %d  skipped %d",
+			p.PagePaths, variants, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
+	}
 	return styleDim.Render(fmt.Sprintf("pages %d  assets %d  errors %d  skipped %d",
-		p.Pages, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
+		p.PagePaths, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
 }
 
 // printSummary prints the final tally and where the mirror landed.
 func printSummary(res clone.Result) {
 	fmt.Fprintln(os.Stderr, styleOK.Render("done")+" "+styleTitle.Render(res.OutDir))
 	fmt.Fprintf(os.Stderr, "  %s %d   %s %d\n",
-		styleAccent.Render("pages"), res.Pages,
+		styleAccent.Render("pages"), res.PagePaths,
 		styleAccent.Render("assets"), res.Assets)
+	if variants := res.Pages - res.PagePaths; variants > 0 {
+		fmt.Fprintf(os.Stderr, "  %s %d\n", styleDim.Render("query variants"), variants)
+	}
+	if res.PagesLinked > 0 {
+		fmt.Fprintf(os.Stderr, "  %s %d\n", styleDim.Render("deduped (linked)"), res.PagesLinked)
+	}
+	if res.AssetSkipped > 0 {
+		fmt.Fprintf(os.Stderr, "  %s %d\n", styleDim.Render("assets over cap (left remote)"), res.AssetSkipped)
+	}
 	if res.PageErrors+res.AssetErrors > 0 {
 		fmt.Fprintf(os.Stderr, "  %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors)
 		printFailures(res)

cli/pack.go

@@ -28,8 +28,13 @@ type packFlags struct {
 	description string
 	language    string
 	date        string
+	incremental bool
 }
 
+// cacheSuffix names the cluster-cache sidecar kage writes next to a packed
+// artifact when --incremental is set.
+const cacheSuffix = ".kagecache"
+
 func newPackCmd() *cobra.Command {
 	f := &packFlags{}
 	cmd := &cobra.Command{
@@ -40,7 +45,9 @@ func newPackCmd() *cobra.Command {
 			"ZIM reader can browse. With --format binary it appends that archive to a copy\n" +
 			"of kage, producing a single executable that serves the site offline when run.\n" +
 			"Add --app to wrap that executable in a double-click desktop app (a .app bundle\n" +
-			"on macOS, an AppImage-style .AppDir on Linux) with the site's favicon as the icon.",
+			"on macOS, an AppImage-style .AppDir on Linux) with the site's favicon as the icon.\n" +
+			"Add --incremental to keep a cache sidecar so re-packing a mirror only compresses\n" +
+			"the clusters that changed, not the whole archive.",
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return runPack(args[0], f)
@@ -53,6 +60,7 @@ func newPackCmd() *cobra.Command {
 	fs.BoolVar(&f.app, "app", false, "wrap the viewer in a double-click desktop app (.app on macOS, .AppImage/.AppDir on Linux)")
 	fs.StringVar(&f.icon, "icon", "", "icon file for --app (default the site's favicon)")
 	fs.BoolVar(&f.noCompress, "no-compress", false, "store every cluster raw, no zstd")
+	fs.BoolVar(&f.incremental, "incremental", false, "reuse compression from a cache sidecar so re-packing a mirror only compresses what changed")
 	fs.StringVar(&f.title, "title", "", "archive title (default the main page's <title>)")
 	fs.StringVar(&f.description, "description", "", "archive description")
 	fs.StringVar(&f.language, "language", "eng", "archive language code")
@@ -80,19 +88,26 @@ func runPack(mirrorArg string, f *packFlags) error {
 
 	switch f.format {
 	case "zim":
-		out, size, err := pack.BuildZIM(dir, zopts)
+		out := f.out
+		if out == "" {
+			out = filepath.Base(dir) + ".zim"
+		}
+		zopts.Out = out
+		var st pack.PackStats
+		if f.incremental {
+			zopts.CachePath = out + cacheSuffix
+			zopts.Stats = &st
+		}
+		outPath, size, err := pack.BuildZIM(dir, zopts)
 		if err != nil {
 			return err
 		}
-		printPackResult(out, size)
-		fmt.Fprintf(os.Stderr, "  open %s\n", styleAccent.Render("kage open "+out))
+		printPackResult(outPath, size)
+		printCacheLine(f.incremental, st)
+		fmt.Fprintf(os.Stderr, "  open %s\n", styleAccent.Render("kage open "+outPath))
 		return nil
 
 	case "binary":
-		zbytes, err := pack.BuildZIMBytes(dir, zopts)
-		if err != nil {
-			return err
-		}
 		target := resolveTargetOS(f.base)
 		out := f.out
 		if out == "" {
@@ -103,11 +118,21 @@ func runPack(mirrorArg string, f *packFlags) error {
 		if target == "windows" && !strings.HasSuffix(strings.ToLower(out), ".exe") {
 			out += ".exe"
 		}
+		var st pack.PackStats
+		if f.incremental {
+			zopts.CachePath = out + cacheSuffix
+			zopts.Stats = &st
+		}
+		zbytes, err := pack.BuildZIMBytes(dir, zopts)
+		if err != nil {
+			return err
+		}
 		path, size, err := pack.BuildBinary(zbytes, pack.BinaryOptions{Out: out, Base: f.base})
 		if err != nil {
 			return err
 		}
 		printPackResult(path, size)
+		printCacheLine(f.incremental, st)
 		printRunHint(path, target)
 		return nil
 
@@ -142,10 +167,16 @@ func runPackApp(dir string, f *packFlags, zopts pack.ZIMOptions) error {
 	if err != nil {
 		return err
 	}
+	var st pack.PackStats
+	if f.incremental {
+		zopts.CachePath = prog + cacheSuffix
+		zopts.Stats = &st
+	}
 	zbytes, err := pack.BuildZIMBytes(dir, zopts)
 	if err != nil {
 		return err
 	}
+	printCacheLine(f.incremental, st)
 
 	switch target {
 	case "darwin":
@@ -345,6 +376,18 @@ func printPackResult(path string, size int64) {
 	fmt.Fprintf(os.Stderr, "  %s %s\n", styleAccent.Render("size"), humanBytes(size))
 }
 
+// printCacheLine reports how much compression the incremental cache reused. On
+// the first pack everything is compressed fresh; on a re-pack after a small
+// change most clusters are reused and only the rest are compressed.
+func printCacheLine(incremental bool, st pack.PackStats) {
+	if !incremental {
+		return
+	}
+	total := st.ClustersReused + st.ClustersCompressed
+	fmt.Fprintf(os.Stderr, "  %s %d reused, %d compressed of %d clusters\n",
+		styleDim.Render("cache"), st.ClustersReused, st.ClustersCompressed, total)
+}
+
 func printRunHint(path, target string) {
 	rel := path
 	if !strings.ContainsAny(path, "/\\") {

cli/parquet.go

@@ -0,0 +1,94 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tamnd/kage/dataset"
+)
+
+// newParquetCmd groups the two columnar conversions: a ZIM archive out to a
+// Parquet table, and a Parquet table back to a ZIM archive. The table is a flat
+// one-row-per-entry shape ready to publish as a dataset, and the round trip is
+// lossless.
+func newParquetCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "parquet",
+		Short: "Convert a ZIM archive to a Parquet dataset and back",
+		Long: "parquet converts a packed ZIM archive into a columnar Parquet table, one row\n" +
+			"per entry with clear columns (url, mime, title, content, extracted text), and\n" +
+			"converts such a table back into a ZIM. The table is the shape a dataset host\n" +
+			"like Hugging Face expects, and the conversion is lossless: a ZIM round-tripped\n" +
+			"through Parquet reproduces every entry, its metadata, and the main page.",
+	}
+	cmd.AddCommand(newParquetExportCmd())
+	cmd.AddCommand(newParquetImportCmd())
+	return cmd
+}
+
+func newParquetExportCmd() *cobra.Command {
+	var out string
+	cmd := &cobra.Command{
+		Use:   "export <file.zim>",
+		Short: "Write a Parquet table from a ZIM archive",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			in := args[0]
+			if out == "" {
+				out = strings.TrimSuffix(in, filepath.Ext(in)) + ".parquet"
+			}
+			st, err := dataset.ZIMToParquet(in, out, Version)
+			if err != nil {
+				return err
+			}
+			printDatasetResult("exported", out)
+			printDatasetStats(st, out)
+			return nil
+		},
+	}
+	cmd.Flags().StringVarP(&out, "out", "o", "", "output path (default <name>.parquet)")
+	return cmd
+}
+
+func newParquetImportCmd() *cobra.Command {
+	var out string
+	cmd := &cobra.Command{
+		Use:   "import <file.parquet>",
+		Short: "Rebuild a ZIM archive from a Parquet table",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			in := args[0]
+			if out == "" {
+				out = strings.TrimSuffix(in, filepath.Ext(in)) + ".zim"
+			}
+			st, err := dataset.ParquetToZIM(in, out, Version)
+			if err != nil {
+				return err
+			}
+			printDatasetResult("imported", out)
+			printDatasetStats(st, out)
+			fmt.Fprintf(os.Stderr, "  open %s\n", styleAccent.Render("kage open "+out))
+			return nil
+		},
+	}
+	cmd.Flags().StringVarP(&out, "out", "o", "", "output path (default <name>.zim)")
+	return cmd
+}
+
+func printDatasetResult(verb, path string) {
+	fmt.Fprintln(os.Stderr, styleOK.Render(verb)+" "+styleTitle.Render(path))
+}
+
+func printDatasetStats(st dataset.Stats, path string) {
+	fmt.Fprintf(os.Stderr, "  %s %d   %s %d\n",
+		styleAccent.Render("rows"), st.Rows,
+		styleAccent.Render("redirects"), st.Redirects)
+	fmt.Fprintf(os.Stderr, "  %s %s content\n", styleDim.Render("content"), humanBytes(st.ContentBytes))
+	if fi, err := os.Stat(path); err == nil {
+		fmt.Fprintf(os.Stderr, "  %s %s\n", styleAccent.Render("size"), humanBytes(fi.Size()))
+	}
+}

cli/root.go

@@ -57,6 +57,7 @@ func newRoot() *cobra.Command {
 	root.AddCommand(newServeCmd())
 	root.AddCommand(newPackCmd())
 	root.AddCommand(newOpenCmd())
+	root.AddCommand(newParquetCmd())
 	return root
 }
 

cli/serve.go

@@ -1,11 +1,13 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/http"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/spf13/cobra"
 )
@@ -24,14 +26,14 @@ func newServeCmd() *cobra.Command {
 			if len(args) == 1 {
 				dir = args[0]
 			}
-			return runServe(dir, addr)
+			return runServe(cmd.Context(), dir, addr)
 		},
 	}
 	cmd.Flags().StringVarP(&addr, "addr", "a", "127.0.0.1:8800", "address to listen on")
 	return cmd
 }
 
-func runServe(dir, addr string) error {
+func runServe(ctx context.Context, dir, addr string) error {
 	info, err := os.Stat(dir)
 	if err != nil {
 		return fmt.Errorf("cannot serve %q: %w", dir, err)
@@ -50,5 +52,24 @@ func runServe(dir, addr string) error {
 	fmt.Fprintln(os.Stderr, styleTitle.Render("kage serve")+" "+styleDim.Render(abs))
 	fmt.Fprintln(os.Stderr, "  open "+styleAccent.Render("http://"+ln.Addr().String()))
 	fmt.Fprintln(os.Stderr, styleDim.Render("  press Ctrl-C to stop"))
-	return srv.Serve(ln)
+
+	srvErr := make(chan error, 1)
+	go func() { srvErr <- srv.Serve(ln) }()
+
+	select {
+	case err := <-srvErr:
+		if err != nil && err != http.ErrServerClosed {
+			return err
+		}
+	case <-ctx.Done():
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		defer cancel()
+		if err := srv.Shutdown(shutdownCtx); err != nil {
+			_ = srv.Close()
+		}
+		if err := <-srvErr; err != nil && err != http.ErrServerClosed {
+			return err
+		}
+	}
+	return nil
 }

clone/cloner.go

@@ -2,6 +2,7 @@ package clone
 
 import (
 	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io"
@@ -18,6 +19,7 @@ import (
 	"github.com/tamnd/kage/sanitize"
 	"github.com/tamnd/kage/urlx"
 	"golang.org/x/net/html"
+	"golang.org/x/time/rate"
 )
 
 // Logf is an optional sink for human-readable progress lines.
@@ -42,9 +44,15 @@ type Cloner struct {
 	mu         sync.Mutex
 	seenAssets map[string]bool
 	enqueued   int // pages offered to the queue
-	wg         sync.WaitGroup
-	pageJobs   chan pageItem
-	assetJobs  chan assetItem
+
+	crawlLimiter *rate.Limiter
+
+	wg        sync.WaitGroup
+	pageJobs  chan pageItem
+	assetJobs chan assetItem
+
+	muContent   sync.Mutex
+	seenContent map[string]string // sha-256 of page bytes -> first path written
 }
 
 type pageItem struct {
@@ -66,19 +74,20 @@ func New(seed *url.URL, cfg Config, logf Logf) *Cloner {
 	host := seed.Hostname()
 	outRoot := cfg.HostDir(host)
 	return &Cloner{
-		cfg:        cfg,
-		seed:       seed,
-		seedHost:   host,
-		outRoot:    outRoot,
-		statePth:   filepath.Join(outRoot, cfg.Reserved, "state.json"),
-		dl:         asset.NewDownloader(cfg.UserAgent, cfg.Timeout, cfg.MaxAssetBytes),
-		httpC:      &http.Client{Timeout: cfg.Timeout},
-		robots:     robots.AllowAll(),
-		front:      newFrontier(),
-		logf:       logf,
-		seenAssets: map[string]bool{},
-		pageJobs:   make(chan pageItem),
-		assetJobs:  make(chan assetItem),
+		cfg:         cfg,
+		seed:        seed,
+		seedHost:    host,
+		outRoot:     outRoot,
+		statePth:    filepath.Join(outRoot, cfg.Reserved, "state.json"),
+		dl:          asset.NewDownloader(cfg.UserAgent, cfg.Timeout, cfg.MaxAssetBytes),
+		httpC:       &http.Client{Timeout: cfg.Timeout},
+		robots:      robots.AllowAll(),
+		front:       newFrontier(),
+		logf:        logf,
+		seenAssets:  map[string]bool{},
+		seenContent: map[string]string{},
+		pageJobs:    make(chan pageItem),
+		assetJobs:   make(chan assetItem),
 	}
 }
 
@@ -98,6 +107,19 @@ func (c *Cloner) assetKey(u *url.URL) string {
 	return urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
 }
 
+// pagePathKey is the identity of a page ignoring its query string, used to tell
+// a real page apart from its ?q=…/?page=… variants for the progress display.
+// Each variant writes its own file (so the crawl stays complete), but they all
+// fold to one path here.
+func (c *Cloner) pagePathKey(u *url.URL) string {
+	if u.RawQuery == "" {
+		return c.pageKey(u)
+	}
+	cp := *u
+	cp.RawQuery = ""
+	return c.pageKey(&cp)
+}
+
 // Run executes the clone until the frontier drains, MaxPages is hit, or ctx is
 // cancelled (which flushes the resume state). It returns the final Result.
 func (c *Cloner) Run(ctx context.Context) (Result, error) {
@@ -126,6 +148,7 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) {
 	defer func() { _ = c.pool.Close() }()
 
 	c.loadRobots(ctx)
+	c.setupCrawlDelayLimiter()
 
 	// Start workers.
 	var workers sync.WaitGroup
@@ -200,6 +223,19 @@ func (c *Cloner) loadRobots(ctx context.Context) {
 	c.robots = robots.Parse(string(data), "kage")
 }
 
+func (c *Cloner) setupCrawlDelayLimiter() {
+	delay := c.cfg.CrawlDelay
+	if delay <= 0 && c.cfg.RespectRobots && c.robots != nil {
+		delay = c.robots.CrawlDelay
+	}
+	if delay <= 0 {
+		c.crawlLimiter = nil
+		return
+	}
+
+	c.crawlLimiter = rate.NewLimiter(rate.Every(delay), 1)
+}
+
 // seedSitemaps adds in-scope sitemap URLs (from robots and the default path) to
 // the frontier.
 func (c *Cloner) seedSitemaps(ctx context.Context) {
@@ -234,9 +270,28 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 		c.stats.skipped.Add(1)
 		return
 	}
+	if !c.waitForCrawlDelay(ctx) {
+		return
+	}
 
 	res, err := c.pool.Render(ctx, j.u.String())
 	if err != nil {
+		var notHTML *browser.ErrNotHTML
+		if errors.As(err, &notHTML) {
+			// The URL is not a page but a file (a zip, a CSV, a bare image) that
+			// reached the page worker through an extensionless link. Hand it to the
+			// asset downloader, where the size and media policy decides whether to
+			// localise it or leave it remote, rather than saving a broken page or
+			// letting Chrome download it to the user's Downloads folder (issue #32).
+			c.front.markVisited(key)
+			if c.wantAsset(j.u) {
+				c.enqueueAsset(ctx, j.u, "")
+				c.logf("not a page, fetching as asset (%s): %s", notHTML.ContentType, j.u.String())
+			} else {
+				c.logf("not a page, left on the live web (%s): %s", notHTML.ContentType, j.u.String())
+			}
+			return
+		}
 		c.failPage(j.u.String(), fmt.Errorf("render: %w", err))
 		return
 	}
@@ -260,6 +315,9 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 			}
 			return u.String() // external page link stays on the live web
 		default: // Asset
+			if !c.wantAsset(u) {
+				return u.String() // off-domain or bulk media: leave it on the live web
+			}
 			c.enqueueAsset(ctx, u, j.u.String())
 			local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
 			return urlx.Rel(fileDir, local)
@@ -268,8 +326,9 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 
 	asset.RewriteHTML(root, j.u, sink)
 	sanitize.CleanTree(root, sanitize.Options{
-		KeepNoscript: c.cfg.KeepNoscript,
-		Banner:       "cloned by kage from " + j.u.String(),
+		KeepNoscript:   c.cfg.KeepNoscript,
+		MobileReadable: c.cfg.MobileReadable,
+		Banner:         "cloned by kage from " + j.u.String(),
 	})
 
 	var buf strings.Builder
@@ -277,12 +336,22 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 		c.failPage(j.u.String(), fmt.Errorf("render html: %w", err))
 		return
 	}
-	if err := c.writeFile(localFile, []byte(buf.String())); err != nil {
+	deduped, err := c.writePage(localFile, []byte(buf.String()))
+	if err != nil {
 		c.failPage(j.u.String(), fmt.Errorf("write %s: %w", localFile, err))
 		return
 	}
 	c.front.markVisited(key)
-	c.stats.pages.Add(1)
+	c.stats.recordPage(c.pagePathKey(j.u), deduped)
+}
+
+// waitForCrawlDelay spaces page render starts.
+func (c *Cloner) waitForCrawlDelay(ctx context.Context) bool {
+	if c.crawlLimiter == nil {
+		return true
+	}
+
+	return c.crawlLimiter.Wait(ctx) == nil
 }
 
 // processAsset downloads one asset, rewriting CSS references on the way, and
@@ -293,6 +362,13 @@ func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
 	}
 	res, err := c.dl.Get(ctx, j.u, j.referer)
 	if err != nil {
+		if errors.Is(err, asset.ErrTooLarge) {
+			// Over the size cap: leave it out rather than save a truncated
+			// fragment. Count it as skipped, not failed, so the run is clean.
+			c.stats.assetSkipped.Add(1)
+			c.logf("asset skipped (over %d MB): %s", c.cfg.MaxAssetBytes>>20, j.u.String())
+			return
+		}
 		c.failAsset(j.u.String(), j.referer, err)
 		return
 	}
@@ -302,6 +378,9 @@ func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
 	if res.IsCSS {
 		fileDir := urlx.Dir(localFile)
 		cssSink := func(u *url.URL, _ urlx.Kind) string {
+			if !c.wantAsset(u) {
+				return u.String() // off-domain or bulk media: leave it on the live web
+			}
 			c.enqueueAsset(ctx, u, j.u.String())
 			local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
 			return urlx.Rel(fileDir, local)
@@ -389,6 +468,22 @@ func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool {
 	return true
 }
 
+// wantAsset reports whether an asset should be downloaded and localized, or left
+// pointing at its live URL. kage skips two classes by default: assets on a host
+// outside the seed's registrable domain (a third-party tracker, an unrelated
+// CDN), and bulk media, installers, and archives whose extension is in the skip
+// set. Both are decisions a page worker can make from the URL alone, before any
+// download, so the rewritten HTML simply keeps the remote link.
+func (c *Cloner) wantAsset(u *url.URL) bool {
+	if c.cfg.AssetSameDomain && !urlx.SameRegistrableDomain(c.seed, u) {
+		return false
+	}
+	if c.cfg.SkipAssetExts[urlx.Ext(u)] {
+		return false
+	}
+	return true
+}
+
 // enqueueAsset offers an asset URL for download, deduping by canonical URL.
 func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) {
 	key := c.assetKey(u)
@@ -410,6 +505,49 @@ func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) {
 	}()
 }
 
+// writePage writes a rendered page, deduping by content. The first page with a
+// given byte content is written normally; a later page with identical bytes is
+// stored as a hard link to that first file, so the same content never occupies
+// disk twice (a faceted site whose ?q=… variants all render the same page is the
+// motivating case). It reports whether this write was deduped. If hard links are
+// unsupported, it falls back to writing the bytes, so correctness never depends
+// on the link succeeding.
+func (c *Cloner) writePage(relSlash string, data []byte) (bool, error) {
+	sum := sha256.Sum256(data)
+	h := string(sum[:])
+
+	c.muContent.Lock()
+	canon, seen := c.seenContent[h]
+	if !seen {
+		c.seenContent[h] = relSlash
+	}
+	c.muContent.Unlock()
+
+	if seen && canon != relSlash {
+		if err := c.linkFile(canon, relSlash); err == nil {
+			return true, nil
+		}
+		// The canonical file may not be on disk yet (a concurrent first write)
+		// or the filesystem may not support links; write the bytes instead.
+	}
+	return false, c.writeFile(relSlash, data)
+}
+
+// linkFile hard-links the already-written canonical file to target, replacing
+// any file already at target. Both paths are confined to the mirror root.
+func (c *Cloner) linkFile(canonSlash, targetSlash string) error {
+	canonFull := filepath.Join(c.outRoot, filepath.FromSlash(canonSlash))
+	targetFull := filepath.Join(c.outRoot, filepath.FromSlash(targetSlash))
+	if !strings.HasPrefix(targetFull, filepath.Clean(c.outRoot)+string(os.PathSeparator)) {
+		return fmt.Errorf("refusing to link outside mirror root: %s", targetSlash)
+	}
+	if err := os.MkdirAll(filepath.Dir(targetFull), 0o755); err != nil {
+		return err
+	}
+	_ = os.Remove(targetFull)
+	return os.Link(canonFull, targetFull)
+}
+
 // writeFile writes data to a slash path relative to the mirror root, creating
 // parent directories. The path is cleaned so it can never escape the root.
 func (c *Cloner) writeFile(relSlash string, data []byte) error {

clone/cloner_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/tamnd/kage/browser"
+	"github.com/tamnd/kage/robots"
 	"github.com/tamnd/kage/urlx"
 )
 
@@ -178,6 +179,56 @@ func TestPageKeyCollapsesDuplicates(t *testing.T) {
 	}
 }
 
+func TestCrawlDelaySpacesPageStarts(t *testing.T) {
+	seed, _ := urlx.ParseSeed("https://ex.com")
+	cfg := DefaultConfig()
+	cfg.RespectRobots = true
+	c := New(seed, cfg, nil)
+	c.robots = &robots.Matcher{CrawlDelay: 20 * time.Millisecond}
+	c.setupCrawlDelayLimiter()
+
+	ctx := context.Background()
+	if !c.waitForCrawlDelay(ctx) {
+		t.Fatal("first crawl-delay wait returned false")
+	}
+
+	start := time.Now()
+	if !c.waitForCrawlDelay(ctx) {
+		t.Fatal("second crawl-delay wait returned false")
+	}
+	if elapsed := time.Since(start); elapsed < 15*time.Millisecond {
+		t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed)
+	}
+}
+
+func TestCrawlDelayFlagOverridesRobots(t *testing.T) {
+	seed, _ := urlx.ParseSeed("https://ex.com")
+	cfg := DefaultConfig()
+	cfg.RespectRobots = true
+	cfg.CrawlDelay = 20 * time.Millisecond
+	c := New(seed, cfg, nil)
+	c.robots = &robots.Matcher{CrawlDelay: time.Minute}
+	c.setupCrawlDelayLimiter()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+	defer cancel()
+	if !c.waitForCrawlDelay(ctx) {
+		t.Fatal("first crawl-delay wait returned false")
+	}
+
+	start := time.Now()
+	if !c.waitForCrawlDelay(ctx) {
+		t.Fatal("second crawl-delay wait returned false")
+	}
+	elapsed := time.Since(start)
+	if elapsed < 15*time.Millisecond {
+		t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed)
+	}
+	if elapsed > 150*time.Millisecond {
+		t.Fatalf("second crawl-delay wait = %v, override likely ignored", elapsed)
+	}
+}
+
 func mustURL(t *testing.T, raw string) *url.URL {
 	t.Helper()
 	u, err := url.Parse(raw)
@@ -261,6 +312,79 @@ func TestCloneRefreshReRenders(t *testing.T) {
 	}
 }
 
+// TestCloneRoutesNonHTMLToAsset guards issue #32: an extensionless link that
+// turns out to be a file (a zip) is classified as a page up front, but once the
+// page worker sees it is not HTML it must be handed to the asset downloader, not
+// saved as a broken page nor downloaded by Chrome to ~/Downloads.
+func TestCloneRoutesNonHTMLToAsset(t *testing.T) {
+	if testing.Short() {
+		t.Skip("clone test drives Chrome; skipped under -short")
+	}
+	if _, ok := browser.LookChrome(); !ok {
+		t.Skip("no Chrome/Chromium found; skipping clone test")
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		// The link has no extension, so it is queued as a page; the server then
+		// answers it with a zip.
+		_, _ = w.Write([]byte(`<!doctype html><html><body>
+<h1>Home</h1><a href="/download">grab the bundle</a></body></html>`))
+	})
+	mux.HandleFunc("/download", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/zip")
+		_, _ = w.Write([]byte("PK\x03\x04 pretend bundle"))
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	seed, err := urlx.ParseSeed(srv.URL)
+	if err != nil {
+		t.Fatalf("parse seed: %v", err)
+	}
+
+	out := t.TempDir()
+	cfg := DefaultConfig()
+	cfg.OutDir = out
+	cfg.Settle = 300 * time.Millisecond
+	cfg.RenderTimeout = 20 * time.Second
+	cfg.Timeout = 10 * time.Second
+
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+
+	res, err := New(seed, cfg, t.Logf).Run(ctx)
+	if err != nil {
+		t.Fatalf("run: %v", err)
+	}
+
+	root := res.OutDir
+	// The home page is a real page and is written.
+	if !fileExists(filepath.Join(root, "index.html")) {
+		t.Error("home page was not written")
+	}
+	// The zip is NOT saved as a page: no download/index.html exists.
+	if fileExists(filepath.Join(root, "download", "index.html")) {
+		t.Error("non-HTML target was saved as a page")
+	}
+	// The zip is fetched as an asset under the reserved tree instead.
+	if res.Assets < 1 {
+		t.Errorf("expected the zip to be fetched as an asset, assets=%d", res.Assets)
+	}
+	assetDir := filepath.Join(root, cfg.Reserved)
+	if !anyFileUnder(t, assetDir, "download") {
+		t.Error("the zip was not downloaded into the reserved asset tree")
+	}
+	if res.PageErrors != 0 {
+		t.Errorf("a rerouted non-HTML target must not count as a page error, got %d", res.PageErrors)
+	}
+}
+
 func readFile(t *testing.T, path string) string {
 	t.Helper()
 	b, err := os.ReadFile(path)

clone/config.go

@@ -36,6 +36,16 @@ type Config struct {
 	Traversal     string
 	MaxAssetBytes int64
 
+	// AssetSameDomain, when set, localizes only assets whose host shares the
+	// seed's registrable domain (apple.com covers developer.apple.com and
+	// www.apple.com but not cdn-apple.com or an unrelated third party). An
+	// off-domain asset is left pointing at its live URL instead of downloaded.
+	AssetSameDomain bool
+	// SkipAssetExts lists asset extensions (".mp4", ".pdf", ".dmg", …) that are
+	// left on the live web rather than downloaded, so bulk media, installers, and
+	// archives do not bloat the mirror. The reference keeps its remote URL.
+	SkipAssetExts map[string]bool
+
 	Timeout       time.Duration // per HTTP request
 	Settle        time.Duration // network-idle quiet period
 	RenderTimeout time.Duration // hard cap per page render
@@ -46,12 +56,14 @@ type Config struct {
 	ScopePrefix       string
 	ExcludePaths      []string
 
-	RespectRobots bool
-	FollowSitemap bool
-	Headless      bool
-	KeepNoscript  bool
-	ChromeBin     string
-	ControlURL    string
+	RespectRobots  bool
+	CrawlDelay     time.Duration // override robots.txt Crawl-delay when > 0
+	FollowSitemap  bool
+	Headless       bool
+	KeepNoscript   bool
+	MobileReadable bool
+	ChromeBin      string
+	ControlURL     string
 
 	// Resume loads the prior run's visited set and skips pages already written,
 	// so an interrupted or repeated clone picks up where it left off instead of
@@ -70,25 +82,52 @@ type Config struct {
 const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +
 	"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
 
+// DefaultSkipAssetExts returns the asset extensions kage leaves on the live web
+// by default: bulk media, installers, and archives that rarely matter for
+// reading a site offline but dominate its download size (a docs site's WWDC
+// videos, .dmg/.pkg installers, and PDF manuals can be most of the bytes).
+// Page-rendering assets (images, fonts, CSS) are deliberately absent, so the
+// offline pages still look right.
+func DefaultSkipAssetExts() map[string]bool {
+	exts := []string{
+		// Video and audio.
+		".mp4", ".m4v", ".mov", ".avi", ".mkv", ".webm", ".flv", ".wmv",
+		".m3u8", ".ts", ".mp3", ".wav", ".flac", ".aac", ".ogg", ".oga",
+		// Installers and disk images.
+		".dmg", ".pkg", ".exe", ".msi", ".deb", ".rpm", ".appimage", ".iso",
+		// Archives.
+		".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".7z", ".rar",
+		// Documents that download rather than render.
+		".pdf",
+	}
+	m := make(map[string]bool, len(exts))
+	for _, e := range exts {
+		m[e] = true
+	}
+	return m
+}
+
 // DefaultConfig returns the baseline configuration.
 func DefaultConfig() Config {
 	return Config{
-		OutDir:        DefaultOutDir(),
-		Reserved:      urlx.DefaultReserved,
-		Workers:       4,
-		AssetWorkers:  8,
-		BrowserPages:  4,
-		MaxAssetBytes: 25 << 20,
-		Traversal:     "bfs",
-		Timeout:       30 * time.Second,
-		Settle:        1500 * time.Millisecond,
-		RenderTimeout: 30 * time.Second,
-		UserAgent:     DefaultUserAgent,
-		RespectRobots: true,
-		FollowSitemap: true,
-		Headless:      true,
-		Resume:        true,
-		Persist:       true,
+		OutDir:          DefaultOutDir(),
+		Reserved:        urlx.DefaultReserved,
+		Workers:         4,
+		AssetWorkers:    8,
+		BrowserPages:    4,
+		MaxAssetBytes:   25 << 20,
+		AssetSameDomain: true,
+		SkipAssetExts:   DefaultSkipAssetExts(),
+		Traversal:       "bfs",
+		Timeout:         30 * time.Second,
+		Settle:          1500 * time.Millisecond,
+		RenderTimeout:   30 * time.Second,
+		UserAgent:       DefaultUserAgent,
+		RespectRobots:   true,
+		FollowSitemap:   true,
+		Headless:        true,
+		Resume:          true,
+		Persist:         true,
 	}
 }
 

clone/dedup_test.go

@@ -0,0 +1,69 @@
+package clone
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestWritePageDedup checks that identical page bytes are stored once and shared
+// by a hard link, while different bytes are written as their own file.
+func TestWritePageDedup(t *testing.T) {
+	dir := t.TempDir()
+	c := &Cloner{outRoot: dir, seenContent: map[string]string{}}
+
+	body := []byte("<html><body>same page</body></html>")
+
+	if deduped, err := c.writePage("a/index.html", body); err != nil || deduped {
+		t.Fatalf("first write: deduped=%v err=%v, want false/nil", deduped, err)
+	}
+	if deduped, err := c.writePage("b/index.html", body); err != nil || !deduped {
+		t.Fatalf("second identical write: deduped=%v err=%v, want true/nil", deduped, err)
+	}
+	if deduped, err := c.writePage("c/index.html", []byte("<html>other</html>")); err != nil || deduped {
+		t.Fatalf("third different write: deduped=%v err=%v, want false/nil", deduped, err)
+	}
+
+	// The two identical pages must be the same file on disk (one inode).
+	fa, err := os.Stat(filepath.Join(dir, "a/index.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	fb, err := os.Stat(filepath.Join(dir, "b/index.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !os.SameFile(fa, fb) {
+		t.Error("identical pages were not hard-linked to the same file")
+	}
+
+	// The different page must stand alone with its own bytes.
+	got, err := os.ReadFile(filepath.Join(dir, "c/index.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != "<html>other</html>" {
+		t.Errorf("third page content = %q", got)
+	}
+}
+
+// TestRecordPageCounts checks that pages counts every write, pagePaths counts
+// distinct query-stripped paths, and pagesLinked counts deduped writes.
+func TestRecordPageCounts(t *testing.T) {
+	var s stats
+	s.recordPage("showcase/index.html", false)
+	s.recordPage("showcase/index.html", true) // a ?q= variant of the same path
+	s.recordPage("showcase/index.html", true)
+	s.recordPage("about/index.html", false)
+
+	p := s.snapshot()
+	if p.Pages != 4 {
+		t.Errorf("Pages = %d, want 4", p.Pages)
+	}
+	if p.PagePaths != 2 {
+		t.Errorf("PagePaths = %d, want 2", p.PagePaths)
+	}
+	if p.PagesLinked != 2 {
+		t.Errorf("PagesLinked = %d, want 2", p.PagesLinked)
+	}
+}

clone/stats.go

@@ -12,16 +12,43 @@ const maxRecordedFailures = 100
 
 // stats are the live counters of a run, read by the CLI's progress ticker.
 type stats struct {
-	pages       atomic.Int64
-	assets      atomic.Int64
-	pageErrors  atomic.Int64
-	assetErrors atomic.Int64
-	skipped     atomic.Int64 // robots-disallowed or out of budget
+	pages        atomic.Int64 // page documents written (one per output file)
+	pagePaths    atomic.Int64 // distinct URL paths among those, ignoring query
+	pagesLinked  atomic.Int64 // pages stored as a hard link to identical content
+	assets       atomic.Int64
+	assetSkipped atomic.Int64 // assets left on the live web (over the size cap)
+	pageErrors   atomic.Int64
+	assetErrors  atomic.Int64
+	skipped      atomic.Int64 // robots-disallowed or out of budget
+
+	muPaths  sync.Mutex
+	seenPath map[string]struct{}
 
 	muFail   sync.Mutex
 	failures []Failure
 }
 
+// recordPage counts a freshly written page. Every write bumps pages; the first
+// write for a given query-stripped path also bumps pagePaths, so the display can
+// separate real pages from the query-string variants (?q=…, ?page=…) that a
+// single path can spawn by the thousand on a faceted site. deduped marks a page
+// whose bytes were stored as a hard link to identical content already on disk.
+func (s *stats) recordPage(pathKey string, deduped bool) {
+	s.pages.Add(1)
+	if deduped {
+		s.pagesLinked.Add(1)
+	}
+	s.muPaths.Lock()
+	if s.seenPath == nil {
+		s.seenPath = make(map[string]struct{})
+	}
+	if _, ok := s.seenPath[pathKey]; !ok {
+		s.seenPath[pathKey] = struct{}{}
+		s.pagePaths.Add(1)
+	}
+	s.muPaths.Unlock()
+}
+
 // Failure is one thing that went wrong, kept for the end-of-run report so the
 // errors are visible as a list rather than only as a count.
 type Failure struct {
@@ -47,22 +74,31 @@ func (s *stats) recordedFailures() []Failure {
 	return out
 }
 
-// Progress is a snapshot of a run for display.
+// Progress is a snapshot of a run for display. Pages is every page document
+// written (it equals the count of HTML files on disk); PagePaths is how many
+// distinct URL paths those represent once query strings are ignored. The
+// difference, Pages-PagePaths, is the number of query-string variants.
 type Progress struct {
-	Pages       int64
-	Assets      int64
-	PageErrors  int64
-	AssetErrors int64
-	Skipped     int64
+	Pages        int64
+	PagePaths    int64
+	PagesLinked  int64
+	Assets       int64
+	AssetSkipped int64
+	PageErrors   int64
+	AssetErrors  int64
+	Skipped      int64
 }
 
 func (s *stats) snapshot() Progress {
 	return Progress{
-		Pages:       s.pages.Load(),
-		Assets:      s.assets.Load(),
-		PageErrors:  s.pageErrors.Load(),
-		AssetErrors: s.assetErrors.Load(),
-		Skipped:     s.skipped.Load(),
+		Pages:        s.pages.Load(),
+		PagePaths:    s.pagePaths.Load(),
+		PagesLinked:  s.pagesLinked.Load(),
+		Assets:       s.assets.Load(),
+		AssetSkipped: s.assetSkipped.Load(),
+		PageErrors:   s.pageErrors.Load(),
+		AssetErrors:  s.assetErrors.Load(),
+		Skipped:      s.skipped.Load(),
 	}
 }
 

clone/wantasset_test.go

@@ -0,0 +1,70 @@
+package clone
+
+import (
+	"testing"
+
+	"github.com/tamnd/kage/urlx"
+)
+
+// TestWantAsset checks the two URL-only skip rules a page worker applies before
+// downloading an asset: off-domain hosts and bulk-media extensions are left on
+// the live web, while the seed's own images, fonts, and stylesheets localize.
+func TestWantAsset(t *testing.T) {
+	seed, _ := urlx.ParseSeed("https://developer.apple.com/")
+	c := New(seed, DefaultConfig(), nil)
+
+	cases := []struct {
+		u    string
+		want bool
+	}{
+		// Same registrable domain: localize.
+		{"https://developer.apple.com/css/main.css", true},
+		{"https://www.apple.com/img/logo.png", true},
+		{"https://images.apple.com/fonts/sf.woff2", true},
+		// Bulk media, installers, archives, and PDFs: leave remote.
+		{"https://developer.apple.com/videos/wwdc.mp4", false},
+		{"https://developer.apple.com/downloads/Xcode.dmg", false},
+		{"https://developer.apple.com/bundle.zip", false},
+		{"https://developer.apple.com/guide.pdf", false},
+		{"https://developer.apple.com/clip.MP4", false}, // case-insensitive
+		// Off-domain hosts: leave remote even for a normal image.
+		{"https://cdn-apple.com/img/x.png", false},
+		{"https://ec.europa.eu/banner.png", false},
+		{"https://mmbiz.qpic.cn/x.jpg", false},
+	}
+	for _, tc := range cases {
+		u := mustURL(t, tc.u)
+		if got := c.wantAsset(u); got != tc.want {
+			t.Errorf("wantAsset(%q) = %v, want %v", tc.u, got, tc.want)
+		}
+	}
+}
+
+// TestWantAssetAllHosts checks that turning the domain scope off localizes a
+// third-party asset, while the media extension rule still applies.
+func TestWantAssetAllHosts(t *testing.T) {
+	seed, _ := urlx.ParseSeed("https://developer.apple.com/")
+	cfg := DefaultConfig()
+	cfg.AssetSameDomain = false
+	c := New(seed, cfg, nil)
+
+	if !c.wantAsset(mustURL(t, "https://cdn-apple.com/img/x.png")) {
+		t.Error("with AssetSameDomain off, an off-domain image should localize")
+	}
+	if c.wantAsset(mustURL(t, "https://cdn-apple.com/video/x.mp4")) {
+		t.Error("a media file should still be skipped regardless of host scope")
+	}
+}
+
+// TestWantAssetKeepMedia checks that an empty skip set (the --keep-media case)
+// localizes media too.
+func TestWantAssetKeepMedia(t *testing.T) {
+	seed, _ := urlx.ParseSeed("https://developer.apple.com/")
+	cfg := DefaultConfig()
+	cfg.SkipAssetExts = map[string]bool{}
+	c := New(seed, cfg, nil)
+
+	if !c.wantAsset(mustURL(t, "https://developer.apple.com/videos/wwdc.mp4")) {
+		t.Error("with an empty skip set, an on-domain video should localize")
+	}
+}

cmd/kage/main.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"os"
 	"os/signal"
+	"syscall"
 
 	"github.com/tamnd/kage/cli"
 	"github.com/tamnd/kage/viewer"
@@ -18,7 +19,13 @@ func main() {
 	// thread; in the default build this is a harmless no-op.
 	viewer.LockMainThread()
 
-	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()
+
+	go func() {
+		<-ctx.Done()
+		stop()
+	}()
+
 	os.Exit(cli.Execute(ctx))
 }

dataset/dataset.go

@@ -0,0 +1,307 @@
+// Package dataset converts between a packed ZIM archive and a columnar Parquet
+// file. The Parquet form is a flat table with one row per archive entry and
+// clear columns (url, mime, title, content, extracted text, redirect target),
+// which is the shape a dataset host such as Hugging Face expects. The conversion
+// is lossless: every entry, its metadata, and the main page survive a ZIM ->
+// Parquet -> ZIM round trip, so the table doubles as an archival representation,
+// not just an export.
+package dataset
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/parquet-go/parquet-go"
+	"golang.org/x/net/html"
+
+	"github.com/tamnd/kage/zim"
+)
+
+// Row is one archive entry as a Parquet record. A content or metadata entry
+// carries its bytes in Content, its type in Mime, and, for HTML, its visible
+// text in Text. A redirect sets IsRedirect and names its destination in
+// RedirectTarget ("<namespace>/<url>", e.g. "C/index.html"), with Content empty.
+//
+// The leading columns (doc_id, url, host, crawl_date, the length pair, text)
+// follow the field names the open-index/open-markdown dataset uses, so a kage
+// export drops into the same tooling other web-crawl datasets on Hugging Face
+// are read with. The trailing columns (namespace, the redirect pair, content)
+// are kage's own: they carry the raw bytes and the ZIM structure that make the
+// table a lossless, reversible copy of the archive rather than a one-way export.
+//
+// Namespace is the single-letter ZIM namespace: "C" for pages and assets, "M"
+// for archive metadata (Title, Description, Language, ...), "W" for well-known
+// entries such as the main-page pointer. Keeping every namespace as a row is
+// what makes the table reversible.
+type Row struct {
+	DocID          string `parquet:"doc_id,dict"`
+	URL            string `parquet:"url"`
+	Host           string `parquet:"host,dict"`
+	Title          string `parquet:"title"`
+	Mime           string `parquet:"mime,dict"`
+	CrawlDate      string `parquet:"crawl_date,dict"`
+	ContentLength  int64  `parquet:"content_length"`
+	TextLength     int64  `parquet:"text_length"`
+	Text           string `parquet:"text"`
+	Namespace      string `parquet:"namespace,dict"`
+	IsRedirect     bool   `parquet:"is_redirect"`
+	RedirectTarget string `parquet:"redirect_target"`
+	Content        []byte `parquet:"content"`
+}
+
+// docNamespace is the UUID v5 namespace for kage doc_id values: the standard
+// URL namespace, matching how open-markdown derives a deterministic id from a
+// page's canonical URL.
+var docNamespace = uuid.NameSpaceURL
+
+// docID returns the deterministic UUID v5 an entry gets in the doc_id column,
+// derived from its host and url so the same page always hashes to the same id
+// across exports. Entries with no host (metadata, well-known) fall back to the
+// namespaced url, which is still stable.
+func docID(host, namespace, url string) string {
+	name := namespace + "/" + url
+	if host != "" {
+		name = host + "/" + url
+	}
+	return uuid.NewSHA1(docNamespace, []byte(name)).String()
+}
+
+// Stats summarises a conversion for the CLI to report.
+type Stats struct {
+	Rows         int64 // total entries written
+	Redirects    int64 // of those, redirects
+	ContentBytes int64 // sum of stored content bytes (uncompressed)
+}
+
+// writeBatch bounds how many rows are buffered before a write to the Parquet
+// writer. It only paces the calls; the writer manages its own row groups.
+const writeBatch = 256
+
+// ZIMToParquet reads the ZIM at zimPath and writes a Parquet table to outPath,
+// one row per entry. The archive's main page is recorded both as its W/mainPage
+// redirect row and as file-level metadata, and a short generator/source line is
+// attached so the dataset is self-describing. version is kage's version string.
+func ZIMToParquet(zimPath, outPath, version string) (Stats, error) {
+	r, err := zim.Open(zimPath)
+	if err != nil {
+		return Stats{}, err
+	}
+	defer func() { _ = r.Close() }()
+
+	f, err := os.Create(outPath)
+	if err != nil {
+		return Stats{}, err
+	}
+	bw := bufio.NewWriter(f)
+	pw := parquet.NewGenericWriter[Row](bw, parquet.Compression(&parquet.Zstd))
+
+	host := metaValue(r, "Source")
+	if host == "" {
+		host = metaValue(r, "Name")
+	}
+	host = strings.ToLower(host)
+	crawlDate := metaValue(r, "Date")
+
+	if ns, u, ok := r.MainPageRef(); ok {
+		pw.SetKeyValueMetadata("kage.main_page", string(ns)+"/"+u)
+	}
+	pw.SetKeyValueMetadata("kage.generator", strings.TrimSpace("kage "+version))
+	pw.SetKeyValueMetadata("kage.source", filepath.Base(zimPath))
+	if host != "" {
+		pw.SetKeyValueMetadata("kage.host", host)
+	}
+
+	var st Stats
+	count := r.Count()
+	batch := make([]Row, 0, writeBatch)
+	flush := func() error {
+		if len(batch) == 0 {
+			return nil
+		}
+		if _, err := pw.Write(batch); err != nil {
+			return err
+		}
+		batch = batch[:0]
+		return nil
+	}
+
+	for i := uint32(0); i < count; i++ {
+		e, err := r.EntryAt(i)
+		if err != nil {
+			return st, fmt.Errorf("read entry %d: %w", i, err)
+		}
+		row := Row{
+			Namespace: string(e.Namespace),
+			URL:       e.URL,
+			Title:     e.Title,
+			Host:      host,
+			CrawlDate: crawlDate,
+			DocID:     docID(host, string(e.Namespace), e.URL),
+		}
+		if e.Redirect {
+			row.IsRedirect = true
+			row.RedirectTarget = string(e.RedirectNamespace) + "/" + e.RedirectURL
+			st.Redirects++
+		} else {
+			row.Mime = e.MimeType
+			row.Content = e.Data
+			row.ContentLength = int64(len(e.Data))
+			st.ContentBytes += int64(len(e.Data))
+			if e.MimeType == "text/html" {
+				row.Text = htmlText(e.Data)
+				row.TextLength = int64(len(row.Text))
+			}
+		}
+		st.Rows++
+		batch = append(batch, row)
+		if len(batch) == cap(batch) {
+			if err := flush(); err != nil {
+				return st, err
+			}
+		}
+	}
+	if err := flush(); err != nil {
+		return st, err
+	}
+	if err := pw.Close(); err != nil {
+		return st, err
+	}
+	if err := bw.Flush(); err != nil {
+		return st, err
+	}
+	return st, f.Close()
+}
+
+// ParquetToZIM reads the Parquet table at parquetPath and writes the ZIM archive
+// it describes to outPath, reproducing every entry, its metadata, and the main
+// page. version is unused for now; it is accepted so the signature matches its
+// sibling and can record provenance later.
+func ParquetToZIM(parquetPath, outPath, _ string) (Stats, error) {
+	f, err := os.Open(parquetPath)
+	if err != nil {
+		return Stats{}, err
+	}
+	defer func() { _ = f.Close() }()
+
+	pr := parquet.NewGenericReader[Row](f)
+	defer func() { _ = pr.Close() }()
+
+	w := zim.NewWriter()
+	var st Stats
+	var mainNS byte
+	var mainURL string
+	haveMain := false
+
+	buf := make([]Row, writeBatch)
+	for {
+		n, readErr := pr.Read(buf)
+		for i := 0; i < n; i++ {
+			row := buf[i]
+			ns := namespaceByte(row.Namespace)
+			if row.IsRedirect {
+				tns, turl := splitTarget(row.RedirectTarget)
+				w.AddRedirect(ns, row.URL, row.Title, tns, turl)
+				if ns == zim.NamespaceWellKnown && row.URL == "mainPage" {
+					mainNS, mainURL, haveMain = tns, turl, true
+				}
+				st.Redirects++
+			} else {
+				w.AddContent(ns, row.URL, row.Title, row.Mime, row.Content)
+				st.ContentBytes += int64(len(row.Content))
+			}
+			st.Rows++
+		}
+		if readErr == io.EOF {
+			break
+		}
+		if readErr != nil {
+			return st, readErr
+		}
+		if n == 0 {
+			break
+		}
+	}
+	if haveMain {
+		w.SetMainPage(mainNS, mainURL)
+	}
+
+	out, err := os.Create(outPath)
+	if err != nil {
+		return st, err
+	}
+	obw := bufio.NewWriter(out)
+	if _, err := w.WriteTo(obw); err != nil {
+		_ = out.Close()
+		return st, err
+	}
+	if err := obw.Flush(); err != nil {
+		_ = out.Close()
+		return st, err
+	}
+	return st, out.Close()
+}
+
+// metaValue reads an M/ metadata entry as a string, returning "" when it is
+// absent. It lets the exporter fill the host and crawl_date columns from the
+// archive's own metadata.
+func metaValue(r *zim.Reader, name string) string {
+	b, err := r.Get(zim.NamespaceMetadata, name)
+	if err != nil {
+		return ""
+	}
+	return string(b.Data)
+}
+
+// namespaceByte turns a one-letter namespace column back into the ZIM byte,
+// defaulting to the content namespace for an unexpectedly empty value.
+func namespaceByte(s string) byte {
+	if s == "" {
+		return zim.NamespaceContent
+	}
+	return s[0]
+}
+
+// splitTarget parses a "<namespace>/<url>" redirect target back into its parts.
+// The namespace is a single byte, so the url is everything after the first two
+// characters; a malformed value yields the content namespace and the raw string.
+func splitTarget(s string) (byte, string) {
+	if len(s) >= 2 && s[1] == '/' {
+		return s[0], s[2:]
+	}
+	return zim.NamespaceContent, s
+}
+
+// htmlText extracts the visible text of an HTML document: the concatenated text
+// nodes outside script, style, and noscript, with runs of whitespace collapsed
+// to single spaces. It is a derived convenience column for dataset consumers and
+// plays no part in the round trip, which reconstructs pages from Content.
+func htmlText(data []byte) string {
+	doc, err := html.Parse(strings.NewReader(string(data)))
+	if err != nil {
+		return ""
+	}
+	var b strings.Builder
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			switch n.Data {
+			case "script", "style", "noscript":
+				return
+			}
+		}
+		if n.Type == html.TextNode {
+			b.WriteString(n.Data)
+			b.WriteByte(' ')
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+	return strings.Join(strings.Fields(b.String()), " ")
+}

dataset/dataset_test.go

@@ -0,0 +1,115 @@
+package dataset
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/tamnd/kage/zim"
+)
+
+// buildZIM writes a small archive with a page, an asset, metadata, and a
+// main-page redirect to a temp file, and returns its path.
+func buildZIM(t *testing.T) string {
+	t.Helper()
+	w := zim.NewWriter()
+	w.AddContent(zim.NamespaceContent, "index.html", "Home",
+		"text/html", []byte("<html><head><title>Home</title></head><body><script>ignore()</script><h1>Hello</h1><p>World</p></body></html>"))
+	w.AddContent(zim.NamespaceContent, "logo.png", "", "image/png", []byte{0x89, 'P', 'N', 'G', 1, 2, 3})
+	w.AddMetadata("Title", "Test Site")
+	w.AddMetadata("Language", "eng")
+	w.SetMainPage(zim.NamespaceContent, "index.html")
+	w.AddRedirect(zim.NamespaceWellKnown, "mainPage", "", zim.NamespaceContent, "index.html")
+
+	path := filepath.Join(t.TempDir(), "site.zim")
+	f, err := os.Create(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := w.WriteTo(f); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+func TestZIMParquetRoundTrip(t *testing.T) {
+	src := buildZIM(t)
+	dir := t.TempDir()
+	pq := filepath.Join(dir, "site.parquet")
+	dst := filepath.Join(dir, "site2.zim")
+
+	exp, err := ZIMToParquet(src, pq, "test")
+	if err != nil {
+		t.Fatalf("ZIMToParquet: %v", err)
+	}
+	if exp.Rows == 0 {
+		t.Fatal("exported zero rows")
+	}
+	if exp.Redirects == 0 {
+		t.Fatal("expected the mainPage redirect to be exported")
+	}
+
+	imp, err := ParquetToZIM(pq, dst, "test")
+	if err != nil {
+		t.Fatalf("ParquetToZIM: %v", err)
+	}
+	if imp.Rows != exp.Rows {
+		t.Fatalf("row count changed across round trip: exported %d, imported %d", exp.Rows, imp.Rows)
+	}
+	if imp.Redirects != exp.Redirects {
+		t.Fatalf("redirect count changed: exported %d, imported %d", exp.Redirects, imp.Redirects)
+	}
+
+	// The rebuilt archive must serve the same content and resolve its main page.
+	r, err := zim.Open(dst)
+	if err != nil {
+		t.Fatalf("open rebuilt zim: %v", err)
+	}
+	defer func() { _ = r.Close() }()
+
+	home, err := r.Get(zim.NamespaceContent, "index.html")
+	if err != nil {
+		t.Fatalf("get index.html: %v", err)
+	}
+	if got := string(home.Data); got != "<html><head><title>Home</title></head><body><script>ignore()</script><h1>Hello</h1><p>World</p></body></html>" {
+		t.Fatalf("page content changed: %q", got)
+	}
+	if home.MimeType != "text/html" {
+		t.Fatalf("page mime changed: %q", home.MimeType)
+	}
+
+	logo, err := r.Get(zim.NamespaceContent, "logo.png")
+	if err != nil {
+		t.Fatalf("get logo.png: %v", err)
+	}
+	if string(logo.Data) != string([]byte{0x89, 'P', 'N', 'G', 1, 2, 3}) {
+		t.Fatal("asset bytes changed across round trip")
+	}
+
+	title, err := r.Get(zim.NamespaceMetadata, "Title")
+	if err != nil {
+		t.Fatalf("get M/Title: %v", err)
+	}
+	if string(title.Data) != "Test Site" {
+		t.Fatalf("metadata changed: %q", string(title.Data))
+	}
+
+	main, err := r.MainPage()
+	if err != nil {
+		t.Fatalf("main page not set after round trip: %v", err)
+	}
+	if main.URL != "index.html" {
+		t.Fatalf("main page url changed: %q", main.URL)
+	}
+}
+
+func TestHTMLTextStripsScript(t *testing.T) {
+	got := htmlText([]byte("<html><body><script>secret()</script><style>.x{}</style><h1>Visible</h1>  <p>Text\nhere</p></body></html>"))
+	want := "Visible Text here"
+	if got != want {
+		t.Fatalf("htmlText = %q, want %q", got, want)
+	}
+}

docs/content/getting-started/installation.md

@@ -1,6 +1,6 @@
 ---
 title: "Installation"
-description: "Install kage from Go, Homebrew, a release archive, a Linux package, or the container image, and point it at a browser."
+description: "Install kage from Go, Homebrew, Scoop, a release archive, a Linux package, or the container image, and point it at a browser."
 weight: 20
 ---
 
@@ -12,10 +12,38 @@ kage is a single binary. Pick whichever channel suits you.
 go install github.com/tamnd/kage/cmd/kage@latest
 ```
 
-## Homebrew
+## Homebrew (macOS)
 
 ```bash
-brew install tamnd/tap/kage
+brew install --cask tamnd/tap/kage
+```
+
+The cask installs the prebuilt macOS binary. On Linux, use the packages below or
+`go install`.
+
+## Scoop (Windows)
+
+```bash
+scoop bucket add tamnd https://github.com/tamnd/scoop-bucket
+scoop install kage
+```
+
+## Linux (apt and dnf)
+
+A signed apt and dnf repository tracks every release, so `apt upgrade` and
+`dnf upgrade` keep kage current.
+
+```bash
+# Debian, Ubuntu
+curl -fsSL https://tamnd.github.io/linux-repo/gpg.key \
+  | sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg
+echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" \
+  | sudo tee /etc/apt/sources.list.d/tamnd.list
+sudo apt update && sudo apt install kage
+
+# Fedora, RHEL
+sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo
+sudo dnf install kage
 ```
 
 ## Release archives and Linux packages
@@ -24,14 +52,14 @@ Every [release](https://github.com/tamnd/kage/releases) attaches `tar.gz`
 archives (and a `.zip` for Windows) for Linux, macOS, Windows, and FreeBSD, plus
 `.deb`, `.rpm`, and `.apk` packages and a `checksums.txt` with a cosign
 signature. Download the one for your platform, extract `kage`, and put it on your
-`PATH`.
+`PATH`. To install a package directly without the repo above:
 
 ```bash
 # Debian/Ubuntu
-sudo dpkg -i kage_*_linux_amd64.deb
+sudo dpkg -i kage_*_amd64.deb
 
 # Fedora/RHEL
-sudo rpm -i kage_*_linux_amd64.rpm
+sudo rpm -i kage-*.x86_64.rpm
 ```
 
 ## Container

docs/content/guides/packing-a-mirror.md

@@ -41,7 +41,18 @@ kage open paulgraham.com.zim            # read it back with kage
 kiwix-serve paulgraham.com.zim          # or serve it with Kiwix at http://localhost
 ```
 
-You can also double-click the file in the [Kiwix desktop app](https://kiwix.org/en/applications/), or load it on Kiwix for Android or iOS to read your mirror on your phone. One caveat: kage writes a structurally valid archive with the standard metadata, but it does not write the full-text search index that Kiwix's own packs ship with, so browsing works everywhere while in-reader search is limited.
+You can also double-click the file in the [Kiwix desktop app](https://kiwix.org/en/applications/), or load it on Kiwix for Android or iOS to read your mirror on your phone. kage writes the metadata the format and `zimcheck` treat as mandatory, including the title, description, and the favicon Kiwix shows as the book icon in its library, so the archive shows up properly rather than as an untitled, iconless entry.
+
+### Searching a packed mirror
+
+Each page is stored under its own real `<title>`, so the search box in Kiwix (and any other ZIM reader) suggests pages by their readable title: type `five` into a paulgraham.com archive and it offers "Five Founders" and "Female Founders", not a filename. This title search works in every reader with no extra index.
+
+What kage does not write is a Xapian full-text index, the separate search database that lets Kiwix match words inside a page's body. Xapian is GPL and kage is MIT, so linking it in would change kage's license; rather than do that, kage leaves full-text search to its own columnar export. `kage parquet export mirror.zim` writes one row per page with a `text` column, which [DuckDB](https://duckdb.org) searches with its `fts` extension or a plain `ILIKE`, fully offline and ranked:
+
+```bash
+kage parquet export paulgraham.com.zim -o paulgraham.parquet
+duckdb -c "SELECT url FROM 'paulgraham.parquet' WHERE text ILIKE '%schlep blindness%'"
+```
 
 ## A self-contained binary
 
@@ -149,4 +160,6 @@ kage pack paulgraham.com \
   --date 2026-06-14
 ```
 
-`--title` defaults to the main page's `<title>`, then the host name. `--date` defaults to today; pass a fixed value for a fully reproducible file. `--no-compress` stores every cluster raw, which packs fastest and lets a reader without zstd open the result. `-o/--out` overrides the output path for either format.
+`--title` defaults to the main page's `<title>`, then the host name. `--description` defaults to a line derived from the host, since a description is mandatory metadata. `--date` defaults to today; pass a fixed value for a fully reproducible file. `--no-compress` stores every cluster raw, which packs fastest and lets a reader without zstd open the result. `-o/--out` overrides the output path for either format.
+
+Beyond these, kage fills in the rest of the mandatory metadata on its own: a unique `Name`, the `Language`, and an `Illustrator_48x48@1` icon. The icon is the site's favicon dug out of the mirror (an `apple-touch-icon.png`, `favicon.png`, or a PNG-based `favicon.ico`) and rescaled to a 48x48 PNG, the same discovery `--app` uses for the desktop icon. A site that ships only a legacy BMP `.ico` or no icon at all is packed without one rather than with a broken image.

docs/content/reference/cli.md

@@ -47,6 +47,7 @@ images, and fonts, and writes a browsable mirror to `<out>/<host>/`.
 | Flag | Default | Meaning |
 |------|---------|---------|
 | `--no-robots` | `false` | Ignore `robots.txt` |
+| `--crawl-delay` | `0s` | Override robots.txt `Crawl-delay` between page starts (0 = use robots.txt) |
 | `--no-sitemap` | `false` | Do not seed URLs from `sitemap.xml` |
 | `--user-agent` | Chrome UA | User-Agent for asset and robots fetches |
 
@@ -69,7 +70,6 @@ images, and fonts, and writes a browsable mirror to `<out>/<host>/`.
 | `--workers` | `4` | Concurrent page render workers |
 | `--asset-workers` | `8` | Concurrent asset download workers |
 | `--browser-pages` | `4` | Chrome page-pool size |
-| `--max-asset-mb` | `25` | Skip assets larger than N MB |
 | `--timeout` | `30s` | Per-request timeout |
 | `-q, --quiet` | `false` | Suppress per-page progress lines |
 
@@ -102,9 +102,11 @@ works right after `kage clone example.com`.
 | `--format` | `zim` | Output format: `zim` or `binary` |
 | `-o, --out` | per format | Output path; `<host>.zim` for zim, `<host>` (or `<host>.exe`) for binary |
 | `--base` | this kage | Base kage binary to append to (`--format binary`); point at another platform's binary to build a viewer for it |
+| `--app` | `false` | Wrap the viewer in a double-click desktop app (`.app` on macOS, `.AppImage`/`.AppDir` on Linux) with the site's favicon as the icon |
+| `--icon` | site favicon | Icon file for `--app`, overriding the favicon found in the mirror |
 | `--no-compress` | `false` | Store every cluster raw, no zstd |
 | `--title` | main page `<title>` | Archive title |
-| `--description` | | Archive description |
+| `--description` | host-derived line | Archive description (mandatory metadata, defaulted when unset) |
 | `--language` | `eng` | Archive language code |
 | `--date` | today | Archive date (`YYYY-MM-DD`); pass a fixed value for a reproducible file |
 

docs/content/reference/configuration.md

@@ -36,7 +36,7 @@ $HOME/data/kage/example.com/
 │   │   ├── logo.png
 │   │   └── fonts/body.woff2
 │   ├── cdn.example.com/        # assets from other hosts, by host
-│   └── state.json              # visited set, for --resume
+│   └── state.json              # visited set, for resume
 └── ...
 ```
 

docs/content/reference/release-notes.md

@@ -6,6 +6,56 @@ weight: 40
 
 The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version.
 
+## v0.3.9
+
+A fix for the antivirus warning some Windows users hit when installing kage.
+
+- **The Windows build no longer ships the leakless helper antivirus flags.** kage renders pages with [go-rod](https://github.com/go-rod/rod), whose launcher pulls in [leakless](https://github.com/ysmood/leakless), a small watchdog that force-kills Chrome if kage exits. leakless carries a prebuilt helper binary for every platform and links the Windows one straight into `kage.exe`. Windows Defender recognises that helper as `Trojan:Win32/Kepavll!rfn` and quarantines it, so a fresh `scoop install` failed with a virus warning on `leakless.exe` ([#68](https://github.com/tamnd/kage/issues/68)). kage already launches Chrome with leakless switched off, so the helper never ran anyway. It is now replaced with a stub that carries no embedded binary, which drops about 1.28 MB from the Windows build and clears the warning. Thanks to John Pywtorak for the report. `go install`, unaffected before, stays clean.
+
+## v0.3.4
+
+Two community fixes: a clean stop for `kage serve`, and pages with heavy JavaScript that used to be dropped.
+
+- **`kage serve` stops on Ctrl-C.** The preview server was started with a blocking call that never watched for an interrupt, so stopping it meant killing the process. kage now shuts the server down gracefully on an interrupt or a `SIGTERM`, with a short timeout before forcing the listener closed. Thanks to Xirui Wang ([#35](https://github.com/tamnd/kage/pull/35)) and Kaidi Zhao ([#38](https://github.com/tamnd/kage/pull/38)).
+- **Pages with deeply nested JavaScript still clone.** Chrome's DevTools Protocol returns "Object reference chain is too long" while loading a page whose script builds a deeply nested object graph, but the page's HTML has already loaded and the error is only about Chrome's internal object tracking. kage now recognises that error and finishes rendering instead of dropping the page ([#36](https://github.com/tamnd/kage/issues/36)). Thanks to Gautam Kumar ([#39](https://github.com/tamnd/kage/pull/39)).
+
+## v0.3.3
+
+A fix for Chrome saving a file to your Downloads folder mid-crawl.
+
+- **A crawl never writes to your Downloads folder.** A link with no file extension is queued as a page, so the page worker opened it in Chrome. When such a link served a binary, a zip or a CSV, Chrome saved the file to `~/Downloads`, a surprise side effect of running a clone ([#32](https://github.com/tamnd/kage/issues/32)). kage now denies Chrome-initiated downloads outright, since every asset is fetched through kage's own downloader and the browser never needs to write a file. As a second layer, kage detects a navigation whose response is not HTML and reroutes that URL to the asset downloader, where the size and media policy decides whether to localise it or leave it on the live web, instead of saving a broken page.
+
+## v0.3.2
+
+A fix for garbled text on pages that did not carry a charset of their own.
+
+- **Saved pages declare UTF-8.** kage writes every page as UTF-8, but a site that set its charset only in the HTTP `Content-Type` header, with no `<meta charset>` in the markup, lost that signal once the page became a standalone file. A reader serving the bytes without a charset fell back to its locale encoding and turned every curly quote, dash, and non-breaking space into mojibake. kage now inserts a `<meta charset="utf-8">` at the top of `<head>` when the page does not already declare one, so the page renders correctly in any reader.
+
+## v0.3.1
+
+A fix for broken styling when a packed mirror's home page is a nested page.
+
+- **`/` redirects to the main page instead of serving it in place.** A page's saved asset links are mirror-relative (`../_kage/...`), computed for that page's own location. The viewer was answering `/` with the main page's bytes directly, so the browser resolved those links against `/` and 404ed the page's CSS and images. A `developer.apple.com/documentation` mirror opened at `/` came up completely unstyled. kage now redirects `/` to the main page's canonical path, the way the archive's `W/mainPage` redirect does, so relative assets resolve correctly. Kiwix already followed that redirect, so it was never affected.
+
+## v0.3.0
+
+Leaner mirrors, and a way to publish one as a dataset. A clone now keeps the assets that make a site readable offline and leaves the bulk downloads on the live web, and a packed archive converts to a columnar table that drops straight into dataset tooling.
+
+- **Bulk downloads stay remote by default.** Video and audio, installers and disk images (`.dmg`, `.pkg`, `.exe`, `.msi`, ...), archives, and PDFs are left pointing at their live URL rather than downloaded, because they are rarely needed to read a site offline yet routinely make up most of its bytes. On a `developer.apple.com` crawl that class was 18 of 19 GB. Page-rendering assets (images, fonts, CSS) are untouched. `--keep-media` restores the old behaviour, and `--skip-ext .foo` leaves more extensions remote.
+- **Assets come only from the site's own domain by default.** Localising is scoped to the seed's registrable domain, so `developer.apple.com` still pulls from `www.apple.com` and `images.apple.com` but not a separate brand domain or an off-topic third party (an embedded tracker, an unrelated CDN). `--all-asset-hosts` downloads from any host as before.
+- **The size cap skips instead of truncating.** An asset over `--max-asset-mb` was being saved as exactly the first N MB of itself, a corrupt fragment that would never play or run. kage now checks the response size and leaves an over-cap asset out of the mirror entirely, pointing at its live URL. On the apple crawl this was about a gigabyte of half-downloaded WWDC videos and `.dmg` installers.
+- **`kage parquet export` and `import`.** A packed ZIM converts to a flat Parquet table, one row per entry with clear columns (`doc_id`, `url`, `host`, `crawl_date`, `mime`, text, content), the shape a dataset host like [Hugging Face](https://huggingface.co) expects and that DuckDB or pandas reads as is. The column names follow the [open-index/open-markdown](https://huggingface.co/datasets/open-index/open-markdown) dataset, with `doc_id` a deterministic UUID v5 of the page URL, so a kage export sits alongside other web-crawl datasets. The conversion is lossless: a ZIM round-tripped through Parquet reproduces every entry, its metadata, and the main page byte for byte.
+- **`kage pack --incremental`.** Packing keeps a small cache sidecar next to the output and reuses the compression of any cluster whose bytes have not changed since the last pack. Compressing clusters with zstd is the dominant cost of packing a large mirror, so re-packing after a small change (a `--refresh`, a handful of edited pages) only compresses what actually changed. A reused cluster is byte-for-byte what a fresh compression produces, so the archive stays deterministic.
+- **Identical pages are stored once.** When a rendered page's bytes match a page already written, kage stores it as a hard link to the first copy instead of a second full file, collapsing the duplicate content a faceted site spawns when many `?q=…`/`?page=…` URLs render the same page. The summary reports how many were deduped.
+- **Cleaner progress counting.** The live counter shows distinct URL paths as "pages" and the query-string permutations one path can spawn separately as "variants", so the number tracks the site's real size instead of being inflated by `?q=…` URLs.
+
+## v0.2.1
+
+Packed ZIM archives now carry the metadata Kiwix expects, so a mirror shows up in a ZIM reader's library with a title, a description, and an icon instead of as a blank entry.
+
+- **Mandatory metadata is always written.** Every archive now gets a `Name` and a `Description` (a line derived from the host when `--description` is not given), the two fields `zimcheck` flags as missing otherwise.
+- **The favicon becomes the book icon.** When the mirror has a usable icon (an `apple-touch-icon.png`, `favicon.png`, or a PNG-based `favicon.ico`), kage rescales it to a 48x48 PNG and stores it as `Illustrator_48x48@1`, which is the icon Kiwix shows for the archive in its library. A site with no usable icon is packed without one rather than with a broken image.
+
 ## v0.2.0
 
 Double-click apps, so a packed mirror opens like a real desktop app instead of a terminal program.

go.mod

@@ -7,14 +7,18 @@ require (
 	github.com/charmbracelet/fang v1.0.0
 	github.com/go-rod/rod v0.116.2
 	github.com/go-rod/stealth v0.4.9
+	github.com/google/uuid v1.6.0
 	github.com/klauspost/compress v1.18.6
+	github.com/parquet-go/parquet-go v0.30.1
 	github.com/spf13/cobra v1.10.2
 	github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6
 	golang.org/x/image v0.42.0
 	golang.org/x/net v0.56.0
+	golang.org/x/time v0.15.0
 )
 
 require (
+	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/charmbracelet/colorprofile v0.3.3 // indirect
 	github.com/charmbracelet/ultraviolet v0.0.0-20251106190538-99ea45596692 // indirect
 	github.com/charmbracelet/x/ansi v0.11.0 // indirect
@@ -33,8 +37,12 @@ require (
 	github.com/muesli/mango-cobra v1.2.0 // indirect
 	github.com/muesli/mango-pflag v0.1.0 // indirect
 	github.com/muesli/roff v0.1.0 // indirect
+	github.com/parquet-go/bitpack v1.0.0 // indirect
+	github.com/parquet-go/jsonlite v1.0.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
+	github.com/twpayne/go-geom v1.6.1 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/ysmood/fetchup v0.2.3 // indirect
 	github.com/ysmood/goob v0.4.0 // indirect
@@ -44,4 +52,13 @@ require (
 	golang.org/x/sync v0.21.0 // indirect
 	golang.org/x/sys v0.46.0 // indirect
 	golang.org/x/text v0.38.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 )
+
+// go-rod's launcher imports github.com/ysmood/leakless, which base64/gzip-embeds
+// a prebuilt leakless.exe into the Windows build. Antivirus engines flag that
+// embedded helper as malware and quarantine kage on install (issue #68). kage
+// always launches Chrome with leakless disabled (browser/leakless.go), so the
+// guard is dead weight; this replace swaps in an API-compatible stub that
+// carries no embedded binary.
+replace github.com/ysmood/leakless => ./third_party/leakless

go.sum

@@ -1,5 +1,13 @@
 charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410 h1:D9PbaszZYpB4nj+d6HTWr1onlmlyuGVNfL9gAi8iB3k=
 charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410/go.mod h1:1qZyvvVCenJO2M1ac2mX0yyiIZJoZmDM4DG4s0udJkU=
+github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
+github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
+github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY=
+github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
 github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY=
 github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E=
 github.com/charmbracelet/colorprofile v0.3.3 h1:DjJzJtLP6/NZ8p7Cgjno0CKGr7wwRJGxWUwh2IyhfAI=
@@ -34,6 +42,12 @@ github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
 github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
 github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4=
 github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
@@ -52,6 +66,14 @@ github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe
 github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0=
 github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8=
 github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig=
+github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
+github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
+github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
+github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
+github.com/parquet-go/parquet-go v0.30.1 h1:Oy6ganNrAdFiVwy7wNmWagfPTWA2X9Z3tVHBc7JtuX8=
+github.com/parquet-go/parquet-go v0.30.1/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
@@ -63,10 +85,14 @@ github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4=
+github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
 github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6 h1:VQpB2SpK88C6B5lPHTuSZKb2Qee1QWwiFlC5CKY4AW0=
 github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6/go.mod h1:yE65LFCeWf4kyWD5re+h4XNvOHJEXOCOuJZ4v8l5sgk=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
 github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
 github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
@@ -81,9 +107,6 @@ github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
 github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
 github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
 github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
-github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
-github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
-github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
@@ -97,6 +120,12 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
 golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
 golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pack/cache.go

@@ -0,0 +1,134 @@
+package pack
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"io"
+	"os"
+	"sort"
+
+	"github.com/tamnd/kage/zim"
+)
+
+// cacheMagic tags the sidecar so a stale or foreign file is recognised and
+// ignored rather than misread. The trailing digit is the format version.
+var cacheMagic = [8]byte{'k', 'a', 'g', 'e', 'z', 'c', 'h', '1'}
+
+// clusterCache is a content-addressed store of compressed ZIM clusters, kept in
+// a sidecar next to the archive. Compressing clusters with zstd is the dominant
+// cost of packing a large mirror, so a re-pack reuses the compression of every
+// cluster whose uncompressed bytes are unchanged and only compresses the rest.
+//
+// A cache hit returns exactly what a fresh compression would have produced, so
+// the archive stays deterministic and valid; the cache only saves CPU. Clusters
+// are keyed by the SHA-256 of their uncompressed data section, which the zim
+// writer assembles before compression.
+type clusterCache struct {
+	prev map[[32]byte][]byte // clusters loaded from the previous pack
+	used map[[32]byte][]byte // clusters touched this pack, written back on save
+
+	reused     int // clusters served from the cache this pack
+	compressed int // clusters compressed fresh this pack
+}
+
+func newClusterCache() *clusterCache {
+	return &clusterCache{prev: map[[32]byte][]byte{}, used: map[[32]byte][]byte{}}
+}
+
+// loadClusterCache reads the sidecar at path. A missing, unreadable, or
+// truncated cache is not an error: packing starts cold and rebuilds it.
+func loadClusterCache(path string) *clusterCache {
+	c := newClusterCache()
+	f, err := os.Open(path)
+	if err != nil {
+		return c
+	}
+	defer func() { _ = f.Close() }()
+
+	r := bufio.NewReader(f)
+	var magic [8]byte
+	if _, err := io.ReadFull(r, magic[:]); err != nil || magic != cacheMagic {
+		return c
+	}
+	var count uint32
+	if err := binary.Read(r, binary.LittleEndian, &count); err != nil {
+		return c
+	}
+	for i := uint32(0); i < count; i++ {
+		var h [32]byte
+		if _, err := io.ReadFull(r, h[:]); err != nil {
+			return newClusterCache() // truncated: drop the partial load
+		}
+		var n uint32
+		if err := binary.Read(r, binary.LittleEndian, &n); err != nil {
+			return newClusterCache()
+		}
+		b := make([]byte, n)
+		if _, err := io.ReadFull(r, b); err != nil {
+			return newClusterCache()
+		}
+		c.prev[h] = b
+	}
+	return c
+}
+
+// Compress returns the stored bytes for an uncompressed cluster: the cached
+// compression when the cluster's bytes are unchanged, a fresh zstd compression
+// on a miss. It is the function handed to zim.Writer.SetCompress.
+func (c *clusterCache) Compress(data []byte) []byte {
+	h := sha256.Sum256(data)
+	if b, ok := c.used[h]; ok {
+		return b
+	}
+	if b, ok := c.prev[h]; ok {
+		c.used[h] = b
+		c.reused++
+		return b
+	}
+	b := zim.Compress(data)
+	c.used[h] = b
+	c.compressed++
+	return b
+}
+
+// save writes the clusters touched this pack to path, replacing the previous
+// sidecar. Only touched clusters are written, so clusters that left the mirror
+// drop out and the cache cannot grow without bound. Entries are sorted by hash
+// so the sidecar itself is reproducible. The write goes through a temp file and
+// a rename so a crash mid-write cannot corrupt an existing cache.
+func (c *clusterCache) save(path string) error {
+	hashes := make([][32]byte, 0, len(c.used))
+	for h := range c.used {
+		hashes = append(hashes, h)
+	}
+	sort.Slice(hashes, func(i, j int) bool {
+		return bytes.Compare(hashes[i][:], hashes[j][:]) < 0
+	})
+
+	tmp := path + ".tmp"
+	f, err := os.Create(tmp)
+	if err != nil {
+		return err
+	}
+	w := bufio.NewWriter(f)
+	_, _ = w.Write(cacheMagic[:])
+	_ = binary.Write(w, binary.LittleEndian, uint32(len(hashes)))
+	for _, h := range hashes {
+		b := c.used[h]
+		_, _ = w.Write(h[:])
+		_ = binary.Write(w, binary.LittleEndian, uint32(len(b)))
+		_, _ = w.Write(b)
+	}
+	if err := w.Flush(); err != nil {
+		_ = f.Close()
+		_ = os.Remove(tmp)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		_ = os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, path)
+}

pack/cache_test.go

@@ -0,0 +1,122 @@
+package pack
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/tamnd/kage/zim"
+)
+
+// TestIncrementalPackReusesCompression checks the core promise of the cache: a
+// second pack of an unchanged mirror compresses nothing, reuses every cluster,
+// and produces a byte-identical archive.
+func TestIncrementalPackReusesCompression(t *testing.T) {
+	host := writeMirror(t)
+	dir := t.TempDir()
+	out := filepath.Join(dir, "example.zim")
+	cache := out + ".kagecache"
+
+	var first PackStats
+	a, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &first})
+	if err != nil {
+		t.Fatalf("first pack: %v", err)
+	}
+	if first.ClustersCompressed == 0 {
+		t.Fatal("first pack reported no clusters compressed")
+	}
+	if first.ClustersReused != 0 {
+		t.Errorf("first pack reused %d clusters, want 0", first.ClustersReused)
+	}
+	if _, err := os.Stat(cache); err != nil {
+		t.Fatalf("cache sidecar not written: %v", err)
+	}
+	bytesA, err := os.ReadFile(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var second PackStats
+	b, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &second})
+	if err != nil {
+		t.Fatalf("second pack: %v", err)
+	}
+	if second.ClustersCompressed != 0 {
+		t.Errorf("second pack compressed %d clusters, want 0 (all reused)", second.ClustersCompressed)
+	}
+	if second.ClustersReused != first.ClustersCompressed {
+		t.Errorf("second pack reused %d clusters, want %d", second.ClustersReused, first.ClustersCompressed)
+	}
+	bytesB, err := os.ReadFile(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(bytesA, bytesB) {
+		t.Error("a cached re-pack produced a different archive than the cold pack")
+	}
+}
+
+// TestIncrementalPackRecompressesChange checks that editing one page forces a
+// fresh compression of the cluster that holds it while the rest are reused, and
+// that the archive still opens and serves the new content.
+func TestIncrementalPackRecompressesChange(t *testing.T) {
+	host := writeMirror(t)
+	dir := t.TempDir()
+	out := filepath.Join(dir, "example.zim")
+	cache := out + ".kagecache"
+
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache}); err != nil {
+		t.Fatalf("first pack: %v", err)
+	}
+
+	// Change one page. Its text cluster must be recompressed.
+	edited := filepath.Join(host, "about", "index.html")
+	if err := os.WriteFile(edited, []byte("<!doctype html><title>About</title><h1>Changed</h1>"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	var st PackStats
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &st}); err != nil {
+		t.Fatalf("second pack: %v", err)
+	}
+	if st.ClustersCompressed == 0 {
+		t.Error("editing a page compressed nothing; expected the changed cluster to be recompressed")
+	}
+
+	r, err := zim.Open(out)
+	if err != nil {
+		t.Fatalf("reopen: %v", err)
+	}
+	defer func() { _ = r.Close() }()
+	about, err := r.Get(zim.NamespaceContent, "about/index.html")
+	if err != nil {
+		t.Fatalf("about page missing after re-pack: %v", err)
+	}
+	if !bytes.Contains(about.Data, []byte("Changed")) {
+		t.Errorf("re-packed page did not carry the edit: %q", about.Data)
+	}
+}
+
+// TestClusterCacheSurvivesCorruptSidecar checks that a damaged cache file is
+// treated as a cold start rather than failing the pack.
+func TestClusterCacheSurvivesCorruptSidecar(t *testing.T) {
+	host := writeMirror(t)
+	dir := t.TempDir()
+	out := filepath.Join(dir, "example.zim")
+	cache := out + ".kagecache"
+	if err := os.WriteFile(cache, []byte("not a real cache file"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	var st PackStats
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &st}); err != nil {
+		t.Fatalf("pack over corrupt cache: %v", err)
+	}
+	if st.ClustersReused != 0 {
+		t.Errorf("corrupt cache should reuse nothing, reused %d", st.ClustersReused)
+	}
+	if st.ClustersCompressed == 0 {
+		t.Error("pack over corrupt cache compressed nothing")
+	}
+}

pack/icon.go

@@ -5,17 +5,19 @@ import (
 	"encoding/binary"
 	"fmt"
 	"image"
+	"image/color"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	// Register the decoders image.Decode dispatches to. A favicon is almost
-	// always one of these; SVG and legacy BMP-in-ICO are handled (or skipped)
-	// explicitly below.
+	// always one of these; SVG is skipped explicitly, and a BMP-in-ICO is
+	// decoded by hand below since the stdlib has no BMP decoder.
 	_ "image/gif"
 	_ "image/jpeg"
-	_ "image/png"
+	"image/png"
 )
 
 // iconNames ranks the file names sites use for their icon, best first. A large
@@ -44,6 +46,22 @@ func FindIcon(mirrorDir string) (image.Image, string, bool) {
 	return nil, "", false
 }
 
+// Favicon48 finds the mirror's icon and renders it to a 48x48 PNG, the form the
+// ZIM Illustrator_48x48@1 metadata takes and the icon Kiwix shows for the book.
+// It returns ok=false when the mirror has no usable icon, in which case the
+// archive simply ships without one rather than failing the pack.
+func Favicon48(mirrorDir string) ([]byte, bool) {
+	img, _, ok := FindIcon(mirrorDir)
+	if !ok {
+		return nil, false
+	}
+	var buf bytes.Buffer
+	if err := png.Encode(&buf, scaleSquare(img, 48)); err != nil {
+		return nil, false
+	}
+	return buf.Bytes(), true
+}
+
 // globIcon returns every file under dir whose base name equals name, nearest the
 // root first. Clones store assets under rewritten paths, so the icon may sit a
 // few directories deep rather than at the mirror root.
@@ -70,21 +88,20 @@ func globIcon(dir, name string) []string {
 func depth(p string) int { return strings.Count(p, string(filepath.Separator)) }
 
 // DecodeIcon reads an icon file into an image. It handles the stdlib raster
-// formats (PNG, JPEG, GIF) directly and unwraps a PNG stored inside a .ico
-// container, which is how modern sites ship a high-resolution favicon.ico. A
-// legacy BMP-only .ico returns an error rather than a mangled image, so the
-// caller falls back to the default icon.
+// formats (PNG, JPEG, GIF) directly and unwraps a .ico container, decoding
+// either the PNG a modern high-resolution favicon embeds or the classic BMP/DIB
+// bitmap older ones (Apple's among them) still ship.
 func DecodeIcon(path string) (image.Image, error) {
 	data, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
 	if isICO(data) {
-		png, err := largestPNGInICO(data)
+		img, err := decodeICO(data)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("pack: decode icon %q: %w", path, err)
 		}
-		data = png
+		return img, nil
 	}
 	img, _, err := image.Decode(bytes.NewReader(data))
 	if err != nil {
@@ -104,13 +121,18 @@ func isICO(data []byte) bool {
 
 var pngMagic = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}
 
-// largestPNGInICO scans an .ico directory for PNG-encoded entries and returns
-// the bytes of the largest one. It ignores BMP entries; if none of the entries
-// are PNG it returns an error.
-func largestPNGInICO(data []byte) ([]byte, error) {
+// decodeICO decodes the best entry in an .ico container. It reads the directory,
+// orders the entries largest first (a bigger source rescales to a cleaner 48x48
+// favicon), and decodes each in turn until one works: a PNG entry through the
+// stdlib, a BMP/DIB entry through decodeICOBMP. It errors only when no entry
+// decodes, so a truly garbage .ico still falls back to the default icon.
+func decodeICO(data []byte) (image.Image, error) {
+	if len(data) < 6 {
+		return nil, fmt.Errorf("pack: .ico too short")
+	}
 	count := int(binary.LittleEndian.Uint16(data[4:6]))
-	var best []byte
-	var bestArea int
+	type entry struct{ w, h, off, size int }
+	var ents []entry
 	for i := 0; i < count; i++ {
 		e := 6 + i*16
 		if e+16 > len(data) {
@@ -128,16 +150,180 @@ func largestPNGInICO(data []byte) ([]byte, error) {
 		if off < 0 || size <= 0 || off+size > len(data) {
 			continue
 		}
-		chunk := data[off : off+size]
-		if !bytes.HasPrefix(chunk, pngMagic) {
-			continue // a BMP/DIB entry; skip it
+		ents = append(ents, entry{w, h, off, size})
+	}
+	sort.SliceStable(ents, func(i, j int) bool { return ents[i].w*ents[i].h > ents[j].w*ents[j].h })
+
+	var firstErr error
+	for _, en := range ents {
+		chunk := data[en.off : en.off+en.size]
+		var (
+			img image.Image
+			err error
+		)
+		if bytes.HasPrefix(chunk, pngMagic) {
+			img, _, err = image.Decode(bytes.NewReader(chunk))
+		} else {
+			img, err = decodeICOBMP(chunk, en.w, en.h)
 		}
-		if w*h > bestArea {
-			bestArea, best = w*h, chunk
+		if err == nil {
+			return img, nil
+		}
+		if firstErr == nil {
+			firstErr = err
 		}
 	}
-	if best == nil {
-		return nil, fmt.Errorf("pack: .ico holds no PNG entry")
+	if firstErr == nil {
+		firstErr = fmt.Errorf("pack: .ico holds no decodable entry")
 	}
-	return best, nil
+	return nil, firstErr
+}
+
+// decodeICOBMP decodes one BMP/DIB icon entry: a BITMAPINFOHEADER, an optional
+// color table, the XOR color bitmap, then a 1-bpp AND transparency mask. The
+// header's height covers both bitmaps stacked, so the real image is half of it.
+// Rows run bottom-up and are padded to a 4-byte boundary. Only uncompressed
+// (BI_RGB) entries are handled, which covers every icon that is not a PNG.
+func decodeICOBMP(p []byte, dirW, dirH int) (image.Image, error) {
+	if len(p) < 40 {
+		return nil, fmt.Errorf("pack: ico bmp header truncated")
+	}
+	headerSize := int(binary.LittleEndian.Uint32(p[0:4]))
+	width := int(int32(binary.LittleEndian.Uint32(p[4:8])))
+	height := int(int32(binary.LittleEndian.Uint32(p[8:12])))
+	bits := int(binary.LittleEndian.Uint16(p[14:16]))
+	compression := binary.LittleEndian.Uint32(p[16:20])
+	clrUsed := int(binary.LittleEndian.Uint32(p[32:36]))
+
+	if compression != 0 {
+		return nil, fmt.Errorf("pack: ico bmp compression %d unsupported", compression)
+	}
+	height /= 2 // drop the AND-mask half to get the picture height
+	if width <= 0 || height <= 0 {
+		width, height = dirW, dirH // fall back to the directory dimensions
+	}
+	if width <= 0 || height <= 0 || width > 1024 || height > 1024 {
+		return nil, fmt.Errorf("pack: ico bmp size %dx%d unreasonable", width, height)
+	}
+
+	off := headerSize
+	if headerSize < 40 || off > len(p) {
+		off = 40
+	}
+
+	// A color table follows the header for the palettized depths (BGRA quads).
+	var palette [][4]byte
+	if bits <= 8 {
+		n := clrUsed
+		if n == 0 {
+			n = 1 << bits
+		}
+		for i := 0; i < n && off+4 <= len(p); i++ {
+			palette = append(palette, [4]byte{p[off], p[off+1], p[off+2], p[off+3]})
+			off += 4
+		}
+	}
+
+	xorStride := ((width*bits + 31) / 32) * 4
+	andStride := ((width + 31) / 32) * 4
+	xor := p[off:]
+	var and []byte
+	if andOff := xorStride * height; andOff < len(xor) {
+		and = xor[andOff:]
+	}
+
+	maskBit := func(mask []byte, rowStart, x int) bool {
+		i := rowStart + x/8
+		return i < len(mask) && (mask[i]>>(7-uint(x%8)))&1 == 1
+	}
+
+	img := image.NewNRGBA(image.Rect(0, 0, width, height))
+	anyAlpha := false
+	for y := 0; y < height; y++ {
+		srcY := height - 1 - y // rows are bottom-up
+		row := srcY * xorStride
+		andRow := srcY * andStride
+		for x := 0; x < width; x++ {
+			var r, g, b uint8
+			a := uint8(255)
+			switch bits {
+			case 32:
+				i := row + x*4
+				if i+4 > len(xor) {
+					continue
+				}
+				b, g, r, a = xor[i], xor[i+1], xor[i+2], xor[i+3]
+				if a != 0 {
+					anyAlpha = true
+				}
+			case 24:
+				i := row + x*3
+				if i+3 > len(xor) {
+					continue
+				}
+				b, g, r = xor[i], xor[i+1], xor[i+2]
+			case 8:
+				i := row + x
+				if i >= len(xor) {
+					continue
+				}
+				if c, ok := paletteAt(palette, int(xor[i])); ok {
+					b, g, r = c[0], c[1], c[2]
+				}
+			case 4:
+				i := row + x/2
+				if i >= len(xor) {
+					continue
+				}
+				idx := xor[i] >> 4
+				if x&1 == 1 {
+					idx = xor[i] & 0x0f
+				}
+				if c, ok := paletteAt(palette, int(idx)); ok {
+					b, g, r = c[0], c[1], c[2]
+				}
+			case 1:
+				i := row + x/8
+				if i >= len(xor) {
+					continue
+				}
+				bit := (xor[i] >> (7 - uint(x%8))) & 1
+				if c, ok := paletteAt(palette, int(bit)); ok {
+					b, g, r = c[0], c[1], c[2]
+				}
+			default:
+				return nil, fmt.Errorf("pack: ico bmp %d-bpp unsupported", bits)
+			}
+			if bits != 32 && maskBit(and, andRow, x) {
+				a = 0 // AND-mask transparency for the non-alpha depths
+			}
+			img.SetNRGBA(x, y, color.NRGBA{R: r, G: g, B: b, A: a})
+		}
+	}
+
+	// A 32-bpp icon whose alpha channel is entirely zero is opaque, not
+	// invisible: it leans on the AND mask instead. Reapply opacity from the mask.
+	if bits == 32 && !anyAlpha {
+		for y := 0; y < height; y++ {
+			srcY := height - 1 - y
+			andRow := srcY * andStride
+			for x := 0; x < width; x++ {
+				c := img.NRGBAAt(x, y)
+				if maskBit(and, andRow, x) {
+					c.A = 0
+				} else {
+					c.A = 255
+				}
+				img.SetNRGBA(x, y, c)
+			}
+		}
+	}
+	return img, nil
+}
+
+func paletteAt(pal [][4]byte, i int) ([4]byte, bool) {
+	if i < 0 || i >= len(pal) {
+		return [4]byte{}, false
+	}
+	return pal[i], true
 }

pack/icon_test.go

@@ -81,9 +81,10 @@ func TestDecodeIconICOWithPNG(t *testing.T) {
 	}
 }
 
-func TestDecodeIconBMPOnlyICOFails(t *testing.T) {
-	// A one-entry .ico whose payload is not a PNG (here just filler) must error
-	// so the caller falls back to the default icon.
+func TestDecodeIconGarbageICOFails(t *testing.T) {
+	// A one-entry .ico whose payload is neither a PNG nor a complete DIB header
+	// (here four filler bytes) must error so the caller falls back to the default
+	// icon rather than crashing.
 	var ico bytes.Buffer
 	leWrite(&ico, uint16(0))
 	leWrite(&ico, uint16(1))
@@ -93,13 +94,89 @@ func TestDecodeIconBMPOnlyICOFails(t *testing.T) {
 	leWrite(&ico, uint16(32))
 	leWrite(&ico, uint32(4))
 	leWrite(&ico, uint32(22))
-	ico.Write([]byte{0x42, 0x4d, 0x00, 0x00}) // "BM" DIB filler
+	ico.Write([]byte{0x42, 0x4d, 0x00, 0x00}) // truncated DIB filler
 	p := filepath.Join(t.TempDir(), "favicon.ico")
 	if err := os.WriteFile(p, ico.Bytes(), 0o644); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := DecodeIcon(p); err == nil {
-		t.Fatal("BMP-only .ico should fail to decode")
+		t.Fatal("garbage .ico should fail to decode")
+	}
+}
+
+// icoWithBMP32 wraps a solid px-by-px 32-bpp BMP/DIB bitmap (the format Apple's
+// favicon.ico uses) in a single-entry .ico container, with an all-opaque AND
+// mask. It exercises the hand-written BMP path without a binary fixture.
+func icoWithBMP32(px int, c color.RGBA) []byte {
+	var dib bytes.Buffer
+	leWrite(&dib, uint32(40))    // biSize
+	leWrite(&dib, int32(px))     // biWidth
+	leWrite(&dib, int32(px*2))   // biHeight (color + mask, stacked)
+	leWrite(&dib, uint16(1))     // biPlanes
+	leWrite(&dib, uint16(32))    // biBitCount
+	leWrite(&dib, uint32(0))     // biCompression = BI_RGB
+	leWrite(&dib, uint32(0))     // biSizeImage
+	leWrite(&dib, int32(0))      // biXPelsPerMeter
+	leWrite(&dib, int32(0))      // biYPelsPerMeter
+	leWrite(&dib, uint32(0))     // biClrUsed
+	leWrite(&dib, uint32(0))     // biClrImportant
+	for i := 0; i < px*px; i++ { // XOR: BGRA, full alpha
+		dib.Write([]byte{c.B, c.G, c.R, 0xff})
+	}
+	andStride := ((px + 31) / 32) * 4
+	dib.Write(make([]byte, andStride*px)) // AND mask, all zero = opaque
+
+	var ico bytes.Buffer
+	leWrite(&ico, uint16(0))
+	leWrite(&ico, uint16(1))
+	leWrite(&ico, uint16(1))
+	ico.WriteByte(byte(px))
+	ico.WriteByte(byte(px))
+	ico.WriteByte(0)
+	ico.WriteByte(0)
+	leWrite(&ico, uint16(1))
+	leWrite(&ico, uint16(32))
+	leWrite(&ico, uint32(dib.Len()))
+	leWrite(&ico, uint32(22))
+	ico.Write(dib.Bytes())
+	return ico.Bytes()
+}
+
+func TestDecodeIconBMP32(t *testing.T) {
+	want := color.RGBA{R: 0x33, G: 0x66, B: 0x99, A: 0xff}
+	p := filepath.Join(t.TempDir(), "favicon.ico")
+	if err := os.WriteFile(p, icoWithBMP32(64, want), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	img, err := DecodeIcon(p)
+	if err != nil {
+		t.Fatalf("DecodeIcon: %v", err)
+	}
+	if img.Bounds().Dx() != 64 || img.Bounds().Dy() != 64 {
+		t.Fatalf("size = %v, want 64x64", img.Bounds())
+	}
+	r, g, b, a := img.At(10, 10).RGBA()
+	if r>>8 != 0x33 || g>>8 != 0x66 || b>>8 != 0x99 || a>>8 != 0xff {
+		t.Errorf("pixel = %02x%02x%02x%02x, want 336699ff", r>>8, g>>8, b>>8, a>>8)
+	}
+}
+
+// A BMP-only .ico now decodes end to end through FindIcon, the path that gives a
+// favicon-only site (such as developer.apple.com) its book icon.
+func TestFindIconDecodesBMPICO(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "favicon.ico"), icoWithBMP32(48, color.RGBA{R: 1, G: 2, B: 3, A: 0xff}), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	img, src, ok := FindIcon(dir)
+	if !ok {
+		t.Fatal("FindIcon should decode a BMP .ico")
+	}
+	if filepath.Base(src) != "favicon.ico" {
+		t.Errorf("src = %s, want favicon.ico", filepath.Base(src))
+	}
+	if img.Bounds().Dx() != 48 {
+		t.Errorf("width = %d, want 48", img.Bounds().Dx())
 	}
 }
 

pack/pack_test.go

@@ -3,6 +3,9 @@ package pack
 import (
 	"bytes"
 	"encoding/binary"
+	"image"
+	"image/color"
+	"image/png"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -134,6 +137,88 @@ func TestBuildZIMDeterministic(t *testing.T) {
 	}
 }
 
+// TestBuildZIMMetadata checks the metadata zimcheck treats as mandatory: a
+// Title, Name, Language, Description, and the Illustrator_48x48@1 favicon. The
+// mirror carries a real PNG favicon so the icon path is exercised end to end.
+func TestBuildZIMMetadata(t *testing.T) {
+	host := writeMirror(t)
+
+	// Drop a 64x64 PNG favicon into the mirror so Favicon48 has something to
+	// find, decode, and rescale to 48x48.
+	icon := image.NewRGBA(image.Rect(0, 0, 64, 64))
+	for y := 0; y < 64; y++ {
+		for x := 0; x < 64; x++ {
+			icon.Set(x, y, color.RGBA{R: 0x33, G: 0x66, B: 0x99, A: 0xff})
+		}
+	}
+	var pngBuf bytes.Buffer
+	if err := png.Encode(&pngBuf, icon); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(host, "favicon.png"), pngBuf.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	out := filepath.Join(t.TempDir(), "meta.zim")
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", Version: "test"}); err != nil {
+		t.Fatalf("BuildZIM: %v", err)
+	}
+	r, err := zim.Open(out)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	defer func() { _ = r.Close() }()
+
+	wantText := map[string]string{
+		"Name":        "example.com",
+		"Language":    "eng",
+		"Description": "Offline mirror of example.com, cloned by kage.",
+	}
+	for k, want := range wantText {
+		e, err := r.Get(zim.NamespaceMetadata, k)
+		if err != nil {
+			t.Errorf("M/%s: %v", k, err)
+			continue
+		}
+		if string(e.Data) != want {
+			t.Errorf("M/%s = %q, want %q", k, e.Data, want)
+		}
+	}
+
+	// Illustrator_48x48@1 is a 48x48 PNG with an image/png MIME.
+	ill, err := r.Get(zim.NamespaceMetadata, "Illustrator_48x48@1")
+	if err != nil {
+		t.Fatalf("M/Illustrator_48x48@1: %v", err)
+	}
+	if ill.MimeType != "image/png" {
+		t.Errorf("illustrator mime = %q, want image/png", ill.MimeType)
+	}
+	img, format, err := image.Decode(bytes.NewReader(ill.Data))
+	if err != nil {
+		t.Fatalf("decode illustrator: %v", err)
+	}
+	if format != "png" {
+		t.Errorf("illustrator format = %q, want png", format)
+	}
+	if b := img.Bounds(); b.Dx() != 48 || b.Dy() != 48 {
+		t.Errorf("illustrator size = %dx%d, want 48x48", b.Dx(), b.Dy())
+	}
+
+	// A caller-supplied description overrides the host-derived default.
+	out2 := filepath.Join(t.TempDir(), "meta2.zim")
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out2, Description: "Custom blurb.", Date: "2026-06-14"}); err != nil {
+		t.Fatalf("BuildZIM: %v", err)
+	}
+	r2, err := zim.Open(out2)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	defer func() { _ = r2.Close() }()
+	if d, err := r2.Get(zim.NamespaceMetadata, "Description"); err != nil || string(d.Data) != "Custom blurb." {
+		t.Errorf("M/Description = %q, %v; want %q", d.Data, err, "Custom blurb.")
+	}
+}
+
 func TestPickMainPage(t *testing.T) {
 	cases := []struct {
 		in   []string
@@ -241,6 +326,24 @@ func TestHandler(t *testing.T) {
 	if code, body := get("/"); code != 200 || !bytes.Contains([]byte(body), []byte("Example Home")) {
 		t.Errorf("GET / = %d %.30q", code, body)
 	}
+
+	// "/" must redirect to the main page's canonical content path, not serve its
+	// bytes in place, so the page's mirror-relative asset URLs resolve against the
+	// right base instead of 404ing.
+	noFollow := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
+		return http.ErrUseLastResponse
+	}}
+	resp, err := noFollow.Get(srv.URL + "/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusFound {
+		t.Errorf("GET / status = %d, want 302", resp.StatusCode)
+	}
+	if loc := resp.Header.Get("Location"); loc != "/index.html" {
+		t.Errorf("GET / Location = %q, want %q", loc, "/index.html")
+	}
 	if code, _ := get("/about/index.html"); code != 200 {
 		t.Errorf("GET /about/index.html = %d", code)
 	}
@@ -251,3 +354,80 @@ func TestHandler(t *testing.T) {
 		t.Errorf("GET missing = %d, want 404", code)
 	}
 }
+
+// titleOf returns the stored entry title for a content url, scanning entries in
+// url order since the reader exposes titles only through EntryAt.
+func titleOf(t *testing.T, r *zim.Reader, url string) string {
+	t.Helper()
+	for i := uint32(0); i < r.Count(); i++ {
+		e, err := r.EntryAt(i)
+		if err != nil {
+			continue
+		}
+		if e.Namespace == zim.NamespaceContent && e.URL == url {
+			return e.Title
+		}
+	}
+	t.Fatalf("no content entry for %q", url)
+	return ""
+}
+
+// TestBuildZIMPageTitles checks that each HTML page entry carries its own
+// <title> (collapsed to one line), that a page with no title falls back to its
+// url, and that a non-HTML asset keeps the url as its title.
+func TestBuildZIMPageTitles(t *testing.T) {
+	root := t.TempDir()
+	host := filepath.Join(root, "example.com")
+	files := map[string]string{
+		"index.html":       "<!doctype html><title>Home</title><h1>Hi</h1>",
+		"essay/index.html": "<!doctype html><title>\n  A Long\n  Title  </title><p>x</p>",
+		"bare/index.html":  "<!doctype html><h1>No title here</h1>",
+		"logo.png":         "\x89PNGfake",
+	}
+	for rel, body := range files {
+		p := filepath.Join(host, filepath.FromSlash(rel))
+		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	out := filepath.Join(t.TempDir(), "titles.zim")
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14"}); err != nil {
+		t.Fatalf("BuildZIM: %v", err)
+	}
+	r, err := zim.Open(out)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	defer func() { _ = r.Close() }()
+
+	cases := map[string]string{
+		"index.html":       "Home",
+		"essay/index.html": "A Long Title", // newlines and runs collapsed to single spaces
+		"bare/index.html":  "bare/index.html",
+		"logo.png":         "logo.png",
+	}
+	for url, want := range cases {
+		if got := titleOf(t, r, url); got != want {
+			t.Errorf("title of %q = %q, want %q", url, got, want)
+		}
+	}
+}
+
+func TestCollapseSpaces(t *testing.T) {
+	cases := map[string]string{
+		"  hello  world ":      "hello world",
+		"line\n\tone\r\n  two": "line one two",
+		"":                     "",
+		"   ":                  "",
+		"single":               "single",
+	}
+	for in, want := range cases {
+		if got := collapseSpaces(in); got != want {
+			t.Errorf("collapseSpaces(%q) = %q, want %q", in, got, want)
+		}
+	}
+}

pack/serve.go

@@ -8,14 +8,24 @@ import (
 	"github.com/tamnd/kage/zim"
 )
 
-// Handler serves a ZIM archive over HTTP. "/" maps to the archive's main page;
-// "/a/b.png" maps to the C/a/b.png content entry. Because the saved HTML's links
-// are mirror-relative paths, and those are exactly the C urls, a click in a
+// Handler serves a ZIM archive over HTTP. "/" redirects to the archive's main
+// page; "/a/b.png" maps to the C/a/b.png content entry. Because the saved HTML's
+// links are mirror-relative paths, and those are exactly the C urls, a click in a
 // served page hits the right entry with no rewriting. A miss is a plain 404.
 func Handler(r *zim.Reader) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		p := strings.TrimPrefix(req.URL.Path, "/")
 		if p == "" {
+			// The main page's saved HTML carries mirror-relative asset URLs
+			// (../_kage/...) computed for its own nested location, so serving its
+			// bytes at "/" would resolve them against the wrong base and 404 the
+			// page's CSS and images. Redirect to the page's canonical content path
+			// instead, the way the archive's W/mainPage redirect does, so the
+			// browser resolves those relative URLs correctly.
+			if ns, url, ok := r.MainPageRef(); ok && ns == zim.NamespaceContent {
+				http.Redirect(w, req, "/"+url, http.StatusFound)
+				return
+			}
 			blob, err := r.MainPage()
 			if err != nil {
 				http.NotFound(w, req)

pack/zim.go

@@ -31,6 +31,24 @@ type ZIMOptions struct {
 	Language    string // M/Language (default "eng")
 	Date        string // M/Date, e.g. "2026-06-14"
 	Version     string // kage version, recorded as M/Scraper
+
+	// CachePath, when set, points at a content-addressed cluster cache sidecar.
+	// Compression of unchanged clusters is reused from it, and the cache is
+	// rewritten after a successful pack. Empty disables the cache. It has no
+	// effect with NoCompress, where nothing is compressed.
+	CachePath string
+	// Stats, when non-nil, is filled with how many clusters were reused from the
+	// cache versus compressed fresh. It lets the caller report incremental gains
+	// without changing the function's return signature.
+	Stats *PackStats
+}
+
+// PackStats reports how a pack reused cached compression. ClustersReused is the
+// number of clusters whose compressed bytes came straight from the cache;
+// ClustersCompressed is the number that were zstd-compressed this run.
+type PackStats struct {
+	ClustersReused     int
+	ClustersCompressed int
 }
 
 // BuildZIM walks mirrorDir, turns every file into a C/ content entry, infers the
@@ -38,7 +56,7 @@ type ZIMOptions struct {
 // redirect, and writes a .zim to opts.Out. It returns the output path and the
 // number of bytes written.
 func BuildZIM(mirrorDir string, opts ZIMOptions) (string, int64, error) {
-	w, err := buildWriter(mirrorDir, opts)
+	w, cache, err := buildWriter(mirrorDir, opts)
 	if err != nil {
 		return "", 0, err
 	}
@@ -60,14 +78,21 @@ func BuildZIM(mirrorDir string, opts ZIMOptions) (string, int64, error) {
 		_ = f.Close()
 		return out, n, err
 	}
-	return out, n, f.Close()
+	if err := f.Close(); err != nil {
+		return out, n, err
+	}
+	if err := persistCache(opts.CachePath, cache); err != nil {
+		return out, n, err
+	}
+	fillStats(opts.Stats, cache)
+	return out, n, nil
 }
 
 // BuildZIMBytes is the buffer-returning sibling of BuildZIM: it runs the same
 // walk and returns the archive in memory, which the binary path appends to a
 // base executable without writing the ZIM to disk first.
 func BuildZIMBytes(mirrorDir string, opts ZIMOptions) ([]byte, error) {
-	w, err := buildWriter(mirrorDir, opts)
+	w, cache, err := buildWriter(mirrorDir, opts)
 	if err != nil {
 		return nil, err
 	}
@@ -75,18 +100,41 @@ func BuildZIMBytes(mirrorDir string, opts ZIMOptions) ([]byte, error) {
 	if _, err := w.WriteTo(&buf); err != nil {
 		return nil, err
 	}
+	if err := persistCache(opts.CachePath, cache); err != nil {
+		return nil, err
+	}
+	fillStats(opts.Stats, cache)
 	return buf.Bytes(), nil
 }
 
+// persistCache writes the cluster cache back to its sidecar after a successful
+// pack. It is a no-op when caching is off.
+func persistCache(path string, c *clusterCache) error {
+	if c == nil || path == "" {
+		return nil
+	}
+	return c.save(path)
+}
+
+// fillStats copies the cache's reuse counters into the caller's PackStats when
+// both are present, so the CLI can report incremental gains.
+func fillStats(dst *PackStats, c *clusterCache) {
+	if dst == nil || c == nil {
+		return
+	}
+	dst.ClustersReused = c.reused
+	dst.ClustersCompressed = c.compressed
+}
+
 // buildWriter does the shared work of both BuildZIM and BuildZIMBytes: it loads
 // every file under mirrorDir into a zim.Writer with metadata and a main page.
-func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) {
+func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, *clusterCache, error) {
 	info, err := os.Stat(mirrorDir)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	if !info.IsDir() {
-		return nil, fmt.Errorf("pack: %q is not a directory", mirrorDir)
+		return nil, nil, fmt.Errorf("pack: %q is not a directory", mirrorDir)
 	}
 
 	w := zim.NewWriter()
@@ -94,6 +142,14 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) {
 		w.SetNoCompress(true)
 	}
 
+	// The cluster cache only helps when clusters are compressed; with NoCompress
+	// there is nothing to reuse, so it is skipped.
+	var cache *clusterCache
+	if opts.CachePath != "" && !opts.NoCompress {
+		cache = loadClusterCache(opts.CachePath)
+		w.SetCompress(cache.Compress)
+	}
+
 	skip := urlx.DefaultReserved + "/state.json"
 	var htmlPages []string
 	counts := map[string]int{}
@@ -114,15 +170,20 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) {
 			return err
 		}
 		mime := MimeForExt(rel)
+		title := ""
 		if mime == "text/html" {
 			htmlPages = append(htmlPages, rel)
+			// Store the page's real <title> on its entry, not the URL path, so a
+			// ZIM reader's search box and suggestions match the readable title.
+			// An empty result leaves the writer to fall back to the url.
+			title = htmlTitleOfBytes(data)
 		}
 		counts[mime]++
-		w.AddContent(zim.NamespaceContent, rel, "", mime, data)
+		w.AddContent(zim.NamespaceContent, rel, title, mime, data)
 		return nil
 	})
 	if walkErr != nil {
-		return nil, walkErr
+		return nil, nil, walkErr
 	}
 
 	main := pickMainPage(htmlPages)
@@ -134,10 +195,11 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) {
 	host := filepath.Base(mirrorDir)
 	title := firstNonEmpty(opts.Title, htmlTitleOf(mirrorDir, main), host)
 	w.AddMetadata("Title", title)
+	w.AddMetadata("Name", host)
 	w.AddMetadata("Language", firstNonEmpty(opts.Language, "eng"))
-	if opts.Description != "" {
-		w.AddMetadata("Description", opts.Description)
-	}
+	// Description is mandatory metadata in the ZIM spec, so it is always written:
+	// the caller's text when given, otherwise a line derived from the host.
+	w.AddMetadata("Description", firstNonEmpty(opts.Description, "Offline mirror of "+host+", cloned by kage."))
 	w.AddMetadata("Creator", "kage")
 	w.AddMetadata("Publisher", "kage")
 	if opts.Date != "" {
@@ -146,7 +208,12 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) {
 	w.AddMetadata("Scraper", strings.TrimSpace("kage "+opts.Version))
 	w.AddMetadata("Source", host)
 	w.AddMetadata("Counter", counterString(counts))
-	return w, nil
+	// Illustrator_48x48@1 is the 48x48 PNG favicon Kiwix shows as the archive's
+	// icon. When the mirror has no usable icon the archive ships without one.
+	if png, ok := Favicon48(mirrorDir); ok {
+		w.AddMetadataBytes("Illustrator_48x48@1", "image/png", png)
+	}
+	return w, cache, nil
 }
 
 // pickMainPage chooses the archive's entry point: the root index if present,
@@ -178,16 +245,30 @@ func htmlTitleOf(mirrorDir, mainURL string) string {
 	if mainURL == "" {
 		return ""
 	}
-	f, err := os.Open(filepath.Join(mirrorDir, filepath.FromSlash(mainURL)))
+	data, err := os.ReadFile(filepath.Join(mirrorDir, filepath.FromSlash(mainURL)))
 	if err != nil {
 		return ""
 	}
-	defer func() { _ = f.Close() }()
-	doc, err := html.Parse(f)
+	return htmlTitleOfBytes(data)
+}
+
+// htmlTitleOfBytes returns the first <title> in the given HTML with surrounding
+// and internal whitespace collapsed, or "" if the bytes do not parse or carry
+// no title. Page entries use it so a ZIM reader shows the page's real title
+// instead of its URL path.
+func htmlTitleOfBytes(data []byte) string {
+	doc, err := html.Parse(bytes.NewReader(data))
 	if err != nil {
 		return ""
 	}
-	return strings.TrimSpace(findTitle(doc))
+	return collapseSpaces(findTitle(doc))
+}
+
+// collapseSpaces trims s and replaces every run of whitespace (including the
+// newlines a wrapped <title> can carry) with a single space, so a title reads
+// as one clean line in a reader's library and search results.
+func collapseSpaces(s string) string {
+	return strings.Join(strings.Fields(s), " ")
 }
 
 // findTitle returns the text of the first <title> element in depth-first order.

sanitize/sanitize.go

@@ -2,9 +2,11 @@
 // the saved page is inert: a photograph, not a program.
 //
 // It parses with golang.org/x/net/html, walks the tree, and deletes scripts,
-// event handlers, javascript: URLs, and the dead preconnect/preload hints that
-// mean nothing offline — while leaving styles, images, fonts, forms, and all
-// semantic markup untouched so the layout survives intact.
+// event handlers, javascript: URLs, downlevel IE conditional comments (which
+// can smuggle a <script> past an element-only walk), and the dead
+// preconnect/preload hints that mean nothing offline — while leaving styles,
+// images, fonts, forms, and all semantic markup untouched so the layout
+// survives intact.
 package sanitize
 
 import (
@@ -27,17 +29,27 @@ type Options struct {
 	// Banner, when non-empty, is inserted as an HTML comment at the top of the
 	// document.
 	Banner string
+	// MobileReadable injects a viewport meta tag and a small CSS block that
+	// makes legacy, font-era sites readable on mobile. It is intended for
+	// archives of 1990s/2000s sites that use <font size="2">, table layouts,
+	// and no viewport declaration — all of which render as microscopic text on
+	// a phone. The injected CSS overrides font sizes, loosens line height, caps
+	// the content width, and hides image-map navigation elements that are
+	// useless offline.
+	MobileReadable bool
 }
 
 // Report counts what was removed, for the run summary and for tests.
 type Report struct {
-	ScriptsRemoved     int
-	HandlersRemoved    int
-	NoscriptRemoved    int
-	NoscriptUnwrapped  int
-	JSURLsNeutralized  int
-	MetaRefreshRemoved int
-	DeadLinksRemoved   int
+	ScriptsRemoved      int
+	HandlersRemoved     int
+	NoscriptRemoved     int
+	NoscriptUnwrapped   int
+	JSURLsNeutralized   int
+	MetaRefreshRemoved  int
+	DeadLinksRemoved    int
+	CondCommentsRemoved int
+	CharsetAdded        bool
 }
 
 // jsURLAttrs are attributes whose value may be a javascript: URL.
@@ -67,6 +79,11 @@ func Strip(doc []byte, opts Options) ([]byte, Report, error) {
 func CleanTree(root *html.Node, opts Options) Report {
 	var rep Report
 	clean(root, opts, &rep)
+	rep.CharsetAdded = ensureCharset(root)
+	if opts.MobileReadable {
+		ensureViewport(root)
+		injectMobileCSS(root)
+	}
 	if opts.Banner != "" {
 		insertBanner(root, opts.Banner)
 	}
@@ -78,6 +95,18 @@ func clean(n *html.Node, opts Options, rep *Report) {
 	var next *html.Node
 	for c := n.FirstChild; c != nil; c = next {
 		next = c.NextSibling
+		if c.Type == html.CommentNode {
+			// A downlevel IE conditional comment (<!--[if lt IE 9]>...<![endif]-->)
+			// parses as one comment whose data holds raw markup — a <script src>
+			// among it. The element walk never sees that script, so drop the whole
+			// comment. Downlevel-revealed content lives in sibling nodes, not the
+			// comment data, so it is untouched.
+			if isConditionalComment(c.Data) {
+				n.RemoveChild(c)
+				rep.CondCommentsRemoved++
+			}
+			continue
+		}
 		if c.Type == html.ElementNode {
 			switch c.DataAtom {
 			case atom.Script:
@@ -175,6 +204,19 @@ func isDeadLink(n *html.Node) bool {
 	return false
 }
 
+// isConditionalComment reports whether a comment's data is a downlevel IE
+// conditional-comment marker. Both the downlevel-hidden form (the whole
+// "[if lt IE 9]>...<![endif]" in one comment) and the two markers of the
+// downlevel-revealed form ("[if gte IE 9]><!" and "<![endif]") match, so the
+// markers are stripped while any revealed content, which sits in sibling
+// nodes, stays.
+func isConditionalComment(data string) bool {
+	d := strings.TrimSpace(data)
+	return strings.HasPrefix(d, "[if") ||
+		strings.HasPrefix(d, "<![endif]") ||
+		strings.HasPrefix(d, "[endif]")
+}
+
 // unwrapNoscript replaces a <noscript> with its content. Because x/net/html
 // parses noscript content as raw text (scripting enabled), the text is
 // re-parsed as a fragment in the parent's context and spliced in before the
@@ -199,6 +241,145 @@ func unwrapNoscript(parent, ns *html.Node) {
 	parent.RemoveChild(ns)
 }
 
+// ensureCharset guarantees the document declares UTF-8, inserting a
+// <meta charset="utf-8"> at the top of <head> when none is present, and reports
+// whether it added one. kage renders every saved page as UTF-8, but a source
+// that set its charset only in the HTTP Content-Type header, with no <meta>
+// charset in the markup, loses that signal once the page is a standalone file.
+// A reader then serving the bytes without a charset falls back to its locale
+// encoding and mojibakes every multibyte character (curly quotes, dashes, a
+// non-breaking space). Declaring the charset in the markup makes the page
+// self-describing in any reader, kage's own viewer and Kiwix alike.
+func ensureCharset(root *html.Node) bool {
+	head := findElement(root, atom.Head)
+	if head == nil {
+		return false
+	}
+	if hasCharsetMeta(head) {
+		return false
+	}
+	meta := &html.Node{
+		Type:     html.ElementNode,
+		Data:     "meta",
+		DataAtom: atom.Meta,
+		Attr:     []html.Attribute{{Key: "charset", Val: "utf-8"}},
+	}
+	// The declaration must precede any content for a reader to honour it, so it
+	// goes first in <head>.
+	head.InsertBefore(meta, head.FirstChild)
+	return true
+}
+
+// hasCharsetMeta reports whether head already declares a character encoding,
+// either as <meta charset="..."> or the older <meta http-equiv="Content-Type"
+// content="...; charset=...">.
+func hasCharsetMeta(head *html.Node) bool {
+	for c := head.FirstChild; c != nil; c = c.NextSibling {
+		if c.Type != html.ElementNode || c.DataAtom != atom.Meta {
+			continue
+		}
+		if attr(c, "charset") != "" {
+			return true
+		}
+		if strings.EqualFold(attr(c, "http-equiv"), "content-type") &&
+			strings.Contains(strings.ToLower(attr(c, "content")), "charset=") {
+			return true
+		}
+	}
+	return false
+}
+
+// findElement returns the first element node of the given atom in document
+// order, or nil if none exists.
+func findElement(n *html.Node, a atom.Atom) *html.Node {
+	if n.Type == html.ElementNode && n.DataAtom == a {
+		return n
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		if found := findElement(c, a); found != nil {
+			return found
+		}
+	}
+	return nil
+}
+
+// mobileCSS is injected when MobileReadable is set. It rewrites font-era
+// HTML for comfortable reading on a phone. Key rules:
+//
+//   - box-sizing:border-box — makes padding predictable in a layout built with
+//     HTML width attributes, so our padding doesn't cause overflow.
+//   - body — no fixed max-width here; let the table rules handle width instead.
+//     overflow-x:hidden catches any stray overflow without a scrollbar.
+//   - font element — overrides in-HTML size/face attributes (e.g. <font size="2">).
+//   - [width],[height] — cancels all HTML attribute widths/heights on every
+//     element (tables, tds, imgs, etc.) so fixed-pixel columns become fluid.
+//   - table — fluid, auto layout, no horizontal scroll.
+//   - td — auto width so a three-column table (nav | spacer | content) collapses
+//     to one usable column once the nav td is hidden.
+//   - img — responsive: never wider than its container.
+//   - td:has(>img[usemap]) — hides the entire nav column td, not just the image
+//     inside it; hiding only the img left a tall empty white box.
+//   - td:has(>img[src*="trans_1x1"]) — hides the 26 px spacer column td (a 1×1
+//     transparent GIF whose only job was spacing in the original table layout).
+const mobileCSS = `*{box-sizing:border-box}` +
+	`:root{font-size:18px}` +
+	`body{margin:0;padding:.75em 1em;line-height:1.7;font-family:Georgia,"Times New Roman",serif;overflow-x:hidden}` +
+	`font{font-size:1rem!important;font-family:inherit!important;color:inherit!important}` +
+	`[width]{width:auto!important;max-width:100%!important}` +
+	`[height]{height:auto!important}` +
+	`table{width:100%!important;max-width:100%!important;table-layout:auto!important;border-collapse:collapse!important;word-break:break-word}` +
+	`td,th{width:auto!important;max-width:100%!important;padding:.35em .5em!important;vertical-align:top!important;overflow-wrap:break-word}` +
+	`img{max-width:100%!important;height:auto!important}` +
+	`img[usemap],map{display:none!important}` +
+	`td:has(>img[usemap]),td:has(>map){display:none!important}` +
+	`img[src*="trans_1x1"],img[src*="spacer"],img[height="1"],img[width="1"]{display:none!important}` +
+	`td:has(>img[src*="trans_1x1"]:only-child),td:has(>img[height="1"]:only-child){display:none!important}`
+
+// ensureViewport inserts <meta name="viewport" content="width=device-width,
+// initial-scale=1"> at the top of <head> when the document does not already
+// carry one. Without it a mobile browser shrinks the page to fit the screen
+// at desktop scale, making text unreadably small regardless of CSS font sizes.
+func ensureViewport(root *html.Node) {
+	head := findElement(root, atom.Head)
+	if head == nil {
+		return
+	}
+	// Check whether a viewport meta already exists.
+	for c := head.FirstChild; c != nil; c = c.NextSibling {
+		if c.Type == html.ElementNode && c.DataAtom == atom.Meta &&
+			strings.EqualFold(attr(c, "name"), "viewport") {
+			return
+		}
+	}
+	meta := &html.Node{
+		Type:     html.ElementNode,
+		Data:     "meta",
+		DataAtom: atom.Meta,
+		Attr: []html.Attribute{
+			{Key: "name", Val: "viewport"},
+			{Key: "content", Val: "width=device-width, initial-scale=1"},
+		},
+	}
+	head.InsertBefore(meta, head.FirstChild)
+}
+
+// injectMobileCSS appends a <style> block containing mobileCSS to <head>.
+// It goes at the end of <head> so it wins specificity ties over any existing
+// inline styles the page already carries.
+func injectMobileCSS(root *html.Node) {
+	head := findElement(root, atom.Head)
+	if head == nil {
+		return
+	}
+	style := &html.Node{
+		Type:     html.ElementNode,
+		Data:     "style",
+		DataAtom: atom.Style,
+	}
+	style.AppendChild(&html.Node{Type: html.TextNode, Data: mobileCSS})
+	head.AppendChild(style)
+}
+
 // insertBanner prepends an HTML comment to the document.
 func insertBanner(root *html.Node, text string) {
 	c := &html.Node{Type: html.CommentNode, Data: " " + text + " "}

sanitize/sanitize_test.go

@@ -107,6 +107,48 @@ func TestKeepNoscriptUnwraps(t *testing.T) {
 	}
 }
 
+func TestConditionalCommentScriptRemoved(t *testing.T) {
+	// A downlevel-hidden IE conditional comment hides a <script src> inside a
+	// single comment node, where an element-only walk never reaches it.
+	in := `<html><head>
+<!--[if lt IE 9]><script src="//oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script><![endif]-->
+</head><body><p>real</p></body></html>`
+	out, rep, err := Strip([]byte(in), Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := string(out)
+	if strings.Contains(s, "<script") || strings.Contains(s, "html5shiv") {
+		t.Errorf("conditional-comment script survived:\n%s", s)
+	}
+	if strings.Contains(s, "[if lt IE 9]") {
+		t.Errorf("conditional comment survived:\n%s", s)
+	}
+	if rep.CondCommentsRemoved != 1 {
+		t.Errorf("CondCommentsRemoved = %d, want 1", rep.CondCommentsRemoved)
+	}
+	if !strings.Contains(s, "<p>real</p>") {
+		t.Errorf("real content must survive:\n%s", s)
+	}
+}
+
+func TestConditionalCommentRevealedContentKept(t *testing.T) {
+	// The downlevel-revealed form shows its content to non-IE browsers; the
+	// content lives in sibling nodes, so only the two markers are stripped.
+	in := `<html><body><!--[if gte IE 9]><!--><span class="modern">keep me</span><!--<![endif]--></body></html>`
+	out, _, err := Strip([]byte(in), Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := string(out)
+	if !strings.Contains(s, `<span class="modern">keep me</span>`) {
+		t.Errorf("revealed content was dropped:\n%s", s)
+	}
+	if strings.Contains(s, "[if") || strings.Contains(s, "<![endif]") {
+		t.Errorf("conditional markers survived:\n%s", s)
+	}
+}
+
 func TestKeepMetaRefreshPlain(t *testing.T) {
 	in := `<html><head><meta http-equiv="refresh" content="5;url=/next"></head><body></body></html>`
 	out, _, err := Strip([]byte(in), Options{KeepMetaRefresh: true})
@@ -124,3 +166,121 @@ func TestKeepMetaRefreshPlain(t *testing.T) {
 		t.Error("JS-target meta refresh must be removed regardless")
 	}
 }
+
+func TestCharsetAddedWhenMissing(t *testing.T) {
+	// A page whose source declared its charset only in the HTTP header has no
+	// <meta charset>. The saved file must gain one so a reader does not fall back
+	// to its locale encoding and mojibake the UTF-8 text.
+	in := `<html><head><title>Quotes</title></head><body><p>` +
+		"“curly” — café</p></body></html>"
+	out, rep, err := Strip([]byte(in), Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !rep.CharsetAdded {
+		t.Error("CharsetAdded = false, want true")
+	}
+	s := string(out)
+	if !strings.Contains(strings.ToLower(s), `<meta charset="utf-8"/>`) {
+		t.Errorf("expected an injected meta charset:\n%s", s)
+	}
+	// It must sit at the very start of <head>, before any content.
+	headIdx := strings.Index(s, "<head>")
+	metaIdx := strings.Index(strings.ToLower(s), "<meta charset")
+	titleIdx := strings.Index(s, "<title>")
+	if headIdx >= metaIdx || metaIdx >= titleIdx {
+		t.Errorf("meta charset must come first in head (head=%d meta=%d title=%d)", headIdx, metaIdx, titleIdx)
+	}
+	// The original bytes are preserved as UTF-8.
+	if !strings.Contains(s, "café") {
+		t.Error("UTF-8 content should be preserved")
+	}
+}
+
+func TestMobileReadableInjectsViewportAndCSS(t *testing.T) {
+	// A paulgraham.com-style page: no viewport, tiny <font size="2"> markup.
+	in := `<html><head><title>Essay</title></head>` +
+		`<body><font size="2" face="verdana"><p>Hello world.</p></font></body></html>`
+	out, _, err := Strip([]byte(in), Options{MobileReadable: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := string(out)
+	if !strings.Contains(s, `name="viewport"`) {
+		t.Error("viewport meta not injected")
+	}
+	if !strings.Contains(s, "width=device-width") {
+		t.Error("viewport content wrong")
+	}
+	if !strings.Contains(s, "font-size:18px") {
+		t.Error("mobile CSS not injected")
+	}
+	if !strings.Contains(s, "font{font-size:1rem") {
+		t.Error("font override not in mobile CSS")
+	}
+	// Fluid table rules must be present.
+	if !strings.Contains(s, "table{width:100%") {
+		t.Error("fluid table rule missing")
+	}
+	// [width] override must be present so fixed HTML width attributes are cancelled.
+	if !strings.Contains(s, "[width]{width:auto") {
+		t.Error("[width] override missing")
+	}
+}
+
+func TestMobileReadableHidesNavColumnTd(t *testing.T) {
+	// paulgraham.com wraps its image-map nav in a <td>. Hiding only the img
+	// leaves a tall empty box; the CSS must also target the containing td.
+	in := `<html><head><title>T</title></head><body>` +
+		`<table><tr>` +
+		`<td><map name="nav"><area shape="rect" coords="0,0,67,21" href="index.html"></map>` +
+		`<img src="nav.gif" width="69" height="357" usemap="#nav"/></td>` +
+		`<td><img src="https://cdn.example.com/trans_1x1.gif" height="1" width="26"/></td>` +
+		`<td><p>Essay text.</p></td>` +
+		`</tr></table>` +
+		`</body></html>`
+	out, _, err := Strip([]byte(in), Options{MobileReadable: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := string(out)
+	// td:has(>img[usemap]) rule must be present so the whole nav column is hidden.
+	if !strings.Contains(s, "td:has(>img[usemap])") {
+		t.Error("td:has(>img[usemap]) rule missing from mobile CSS")
+	}
+	// td:has(>img[src*="trans_1x1"]) rule must be present for spacer column.
+	if !strings.Contains(s, `td:has(>img[src*="trans_1x1"]`) {
+		t.Error(`td:has(>img[src*="trans_1x1"]) spacer-column rule missing from mobile CSS`)
+	}
+}
+
+func TestMobileReadableSkipsExistingViewport(t *testing.T) {
+	in := `<html><head><meta name="viewport" content="width=device-width"><title>x</title></head><body></body></html>`
+	out, _, err := Strip([]byte(in), Options{MobileReadable: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n := strings.Count(string(out), `name="viewport"`); n != 1 {
+		t.Errorf("viewport injected when one already existed (count %d)", n)
+	}
+}
+
+func TestCharsetNotDuplicated(t *testing.T) {
+	// A page that already declares a charset, in either form, is left alone.
+	cases := []string{
+		`<html><head><meta charset="utf-8"><title>x</title></head><body></body></html>`,
+		`<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>x</title></head><body></body></html>`,
+	}
+	for _, in := range cases {
+		out, rep, err := Strip([]byte(in), Options{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if rep.CharsetAdded {
+			t.Errorf("CharsetAdded = true for a page that already declares one:\n%s", in)
+		}
+		if n := strings.Count(strings.ToLower(string(out)), "charset"); n != 1 {
+			t.Errorf("charset count = %d, want 1:\n%s", n, out)
+		}
+	}
+}

third_party/leakless/go.mod

@@ -0,0 +1,3 @@
+module github.com/ysmood/leakless
+
+go 1.26.4

third_party/leakless/leakless.go

@@ -0,0 +1,55 @@
+// Package leakless is kage's drop-in replacement for
+// github.com/ysmood/leakless, wired in through a replace directive in the root
+// go.mod.
+//
+// The upstream package guards a child process by extracting a small helper
+// executable that force-kills the child when the parent dies. It ships that
+// helper by base64/gzip-embedding a prebuilt binary for every target
+// (bin_amd64_windows.go and friends), so the packed leakless.exe ends up linked
+// into any program that imports the package, kage included. Antivirus engines
+// flag that embedded Windows helper as malware, so a fresh install of kage got
+// quarantined before it ever ran (issue #68).
+//
+// kage already launches Chrome with leakless disabled (see
+// browser/leakless.go), so the guard is never used. This stub keeps the exact
+// public surface go-rod's launcher depends on (New, Support, LockPort, and the
+// Launcher type's Command/Pid/Err) while carrying no embedded binary, which
+// removes the false positive entirely. Support reports no guard is available,
+// so go-rod's launcher never takes the leakless path even if a caller asked
+// for it.
+package leakless
+
+import "os/exec"
+
+// Launcher mirrors the upstream type. The channel is left unbuffered and is
+// never written to, matching the "may never receive the pid" contract go-rod
+// already tolerates.
+type Launcher struct {
+	pid chan int
+}
+
+// New returns a Launcher. It allocates nothing beyond the pid channel.
+func New() *Launcher {
+	return &Launcher{pid: make(chan int)}
+}
+
+// Command builds the command without a guard wrapper. Because Support returns
+// false, go-rod never calls this in practice; if some other caller did, running
+// the target directly is the correct no-guard behaviour.
+func (l *Launcher) Command(name string, arg ...string) *exec.Cmd {
+	return exec.Command(name, arg...)
+}
+
+// Pid returns the (never-signalled) pid channel.
+func (l *Launcher) Pid() chan int { return l.pid }
+
+// Err returns the guard error, always empty here since there is no guard.
+func (l *Launcher) Err() string { return "" }
+
+// Support reports whether a guard binary is available. It always returns false
+// so callers skip leakless entirely.
+func Support() bool { return false }
+
+// LockPort is the cross-process mutex the upstream guard uses to serialise
+// extraction. With no guard there is nothing to serialise, so it is a no-op.
+func LockPort(port int) func() { return func() {} }

urlx/urlx.go

@@ -19,6 +19,8 @@ import (
 	"net/url"
 	"path"
 	"strings"
+
+	"golang.org/x/net/publicsuffix"
 )
 
 // Kind distinguishes a crawlable page from a downloadable asset; the two map to
@@ -142,9 +144,59 @@ func canonical(u *url.URL, root bool) *url.URL {
 		}
 		c.Path = cleaned
 	}
+	c.RawQuery = safeRawQuery(c.RawQuery)
 	return &c
 }
 
+// safeRawQuery percent-encodes the characters a query string is not allowed to
+// carry on the wire (a space, a control byte, a non-ASCII byte) while leaving
+// everything already legal untouched, including existing %XX escapes and the
+// query sub-delimiters a cache-buster relies on (& = , ; etc.). It exists
+// because real sites emit hrefs with raw spaces in the query, e.g. a CSS link
+// busted with a date string like "?Thursday, 26-Feb-2026 16:26:41 UTC"; a
+// browser encodes the spaces before requesting, but url.Parse keeps RawQuery
+// verbatim, so without this the request line is malformed and the server
+// answers 400. Re-encoding here, on the canonical URL, fixes both the fetch and
+// the on-disk key in one place.
+func safeRawQuery(raw string) string {
+	if raw == "" {
+		return ""
+	}
+	var b strings.Builder
+	for i := 0; i < len(raw); i++ {
+		ch := raw[i]
+		switch {
+		case ch == '%' && i+2 < len(raw) && isHex(raw[i+1]) && isHex(raw[i+2]):
+			// Already a percent-escape; copy it through unchanged.
+			b.WriteString(raw[i : i+3])
+			i += 2
+		case queryByteAllowed(ch):
+			b.WriteByte(ch)
+		default:
+			b.WriteByte('%')
+			const hexDigits = "0123456789ABCDEF"
+			b.WriteByte(hexDigits[ch>>4])
+			b.WriteByte(hexDigits[ch&0x0f])
+		}
+	}
+	return b.String()
+}
+
+// queryByteAllowed reports whether ch may appear literally in a URL query.
+// It is the RFC 3986 query grammar (pchar / "/" / "?"), which keeps the
+// sub-delims that structure a query and rejects spaces, quotes, and controls.
+func queryByteAllowed(ch byte) bool {
+	switch {
+	case ch >= 'a' && ch <= 'z', ch >= 'A' && ch <= 'Z', ch >= '0' && ch <= '9':
+		return true
+	}
+	return strings.IndexByte("-._~!$&'()*+,;=:@/?", ch) >= 0
+}
+
+func isHex(ch byte) bool {
+	return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')
+}
+
 // Key is the canonical string form used to dedup pages and assets.
 func Key(u *url.URL) string { return u.String() }
 
@@ -168,6 +220,34 @@ func SameSite(seed, u *url.URL, allowSub bool) bool {
 	return false
 }
 
+// SameRegistrableDomain reports whether u shares the seed's registrable domain
+// (its eTLD+1). It is looser than SameSite: developer.apple.com, www.apple.com,
+// and images.apple.com all fold to apple.com and count as same-domain, while a
+// separate brand like cdn-apple.com or an unrelated third party like
+// ec.europa.eu does not. It is how kage decides whether an asset host is "the
+// site's own" without listing every subdomain a CDN might use. When either host
+// has no registrable domain (an IP, an oddball TLD), it falls back to an exact
+// host match so the decision stays conservative.
+func SameRegistrableDomain(seed, u *url.URL) bool {
+	sh, uh := seed.Hostname(), u.Hostname()
+	if sh == uh {
+		return true
+	}
+	sd, err1 := publicsuffix.EffectiveTLDPlusOne(sh)
+	ud, err2 := publicsuffix.EffectiveTLDPlusOne(uh)
+	if err1 != nil || err2 != nil {
+		return false
+	}
+	return sd == ud
+}
+
+// Ext returns the lowercased file extension of a URL's last path segment,
+// including the leading dot (".pdf", ".mp4"), or "" when there is none. It
+// ignores the query string, so "/a/clip.mp4?v=2" reports ".mp4".
+func Ext(u *url.URL) string {
+	return strings.ToLower(path.Ext(lastSegment(u.Path)))
+}
+
 // InScope reports whether u should be crawled as a page given the seed and cfg.
 func InScope(seed, u *url.URL, cfg ScopeConfig) bool {
 	if !SameSite(seed, u, cfg.IncludeSubdomains) {

urlx/urlx_test.go

@@ -61,6 +61,12 @@ func TestNormalize(t *testing.T) {
 		{"//cdn.io/x.css", "https://cdn.io/x.css", false},
 		{"HTTPS://Other.COM/Y", "https://other.com/Y", false},
 		{"sub/", "https://ex.com/docs/sub/", false},
+		// A query busted with a raw date string must come back with its spaces
+		// percent-encoded so the request line is valid, while the commas and
+		// colons a query legally carries stay as they are.
+		{"a.css?Thursday, 26-Feb-2026 16:26:41 UTC", "https://ex.com/docs/a.css?Thursday,%2026-Feb-2026%2016:26:41%20UTC", false},
+		{"b.css?v=1&t=a b", "https://ex.com/docs/b.css?v=1&t=a%20b", false},
+		{"c.css?x=%20done", "https://ex.com/docs/c.css?x=%20done", false},
 		{"#top", "", true},
 		{"javascript:void(0)", "", true},
 		{"mailto:a@b.com", "", true},
@@ -110,6 +116,40 @@ func TestInScope(t *testing.T) {
 	}
 }
 
+func TestSameRegistrableDomain(t *testing.T) {
+	seed := mustParse(t, "https://developer.apple.com/")
+	cases := map[string]bool{
+		"https://developer.apple.com/x": true,
+		"https://www.apple.com/x":       true,
+		"https://images.apple.com/x":    true,
+		"https://apple.com/x":           true,
+		"https://cdn-apple.com/x":       false,
+		"https://ec.europa.eu/x":        false,
+		"https://mmbiz.qpic.cn/x":       false,
+	}
+	for u, want := range cases {
+		if got := SameRegistrableDomain(seed, mustParse(t, u)); got != want {
+			t.Errorf("SameRegistrableDomain(%q) = %v, want %v", u, got, want)
+		}
+	}
+}
+
+func TestExt(t *testing.T) {
+	cases := map[string]string{
+		"https://ex.com/a/clip.mp4":     ".mp4",
+		"https://ex.com/a/clip.MP4?v=2": ".mp4",
+		"https://ex.com/style.css":      ".css",
+		"https://ex.com/docs/":          "",
+		"https://ex.com/docs":           "",
+		"https://ex.com/Xcode_15.0.dmg": ".dmg",
+	}
+	for u, want := range cases {
+		if got := Ext(mustParse(t, u)); got != want {
+			t.Errorf("Ext(%q) = %q, want %q", u, got, want)
+		}
+	}
+}
+
 func TestLikelyPage(t *testing.T) {
 	cases := map[string]bool{
 		"https://ex.com/docs":      true,

zim/codec.go

@@ -26,6 +26,12 @@ func zstdEncode(p []byte) []byte {
 	return zstdEnc.EncodeAll(p, nil)
 }
 
+// Compress zstd-compresses p with the exact codec the writer uses for its
+// clusters. It is exported so a caller can cache cluster compression across
+// packs and feed the result back through Writer.SetCompress; a cached cluster is
+// then byte-for-byte what a fresh compression would have produced.
+func Compress(p []byte) []byte { return zstdEncode(p) }
+
 func zstdDecode(p []byte) ([]byte, error) {
 	initZstd()
 	return zstdDec.DecodeAll(p, nil)

zim/reader.go

@@ -108,6 +108,67 @@ func (r *Reader) MainPage() (Blob, error) {
 	return r.blobAtIndex(r.hdr.mainPage, 0)
 }
 
+// Entry is one directory entry as stored, returned by EntryAt. A redirect keeps
+// Data nil and names its target in RedirectNamespace/RedirectURL; any other
+// entry carries its bytes in Data and its type in MimeType. Unlike Get, EntryAt
+// does not follow redirects, so a caller can round-trip every entry, the
+// redirects included.
+type Entry struct {
+	Namespace         byte
+	URL               string
+	Title             string
+	MimeType          string
+	Redirect          bool
+	RedirectNamespace byte
+	RedirectURL       string
+	Data              []byte
+}
+
+// EntryAt returns the directory entry at idx, where 0 <= idx < Count, in the
+// archive's URL order. It is the iteration counterpart to Get: it exposes every
+// entry exactly as stored, including metadata and redirects, which is what an
+// exporter needs to reproduce the archive.
+func (r *Reader) EntryAt(idx uint32) (Entry, error) {
+	d, err := r.direntAtIndex(idx)
+	if err != nil {
+		return Entry{}, err
+	}
+	e := Entry{Namespace: d.namespace, URL: d.url, Title: d.title}
+	if d.redirect {
+		e.Redirect = true
+		td, err := r.direntAtIndex(d.targetIndex)
+		if err != nil {
+			return Entry{}, fmt.Errorf("zim: redirect target of %c/%s: %w", d.namespace, d.url, err)
+		}
+		e.RedirectNamespace = td.namespace
+		e.RedirectURL = td.url
+		return e, nil
+	}
+	data, err := r.blobData(d.cluster, d.blob)
+	if err != nil {
+		return Entry{}, err
+	}
+	if int(d.mimeIdx) < len(r.mimes) {
+		e.MimeType = r.mimes[d.mimeIdx]
+	}
+	e.Data = data
+	return e, nil
+}
+
+// MainPageRef returns the namespace and url of the archive's entry point and
+// whether one is set, so an exporter can record which entry is the main page
+// without following the W/mainPage redirect.
+func (r *Reader) MainPageRef() (byte, string, bool) {
+	if r.hdr.mainPage == noMainPage {
+		return 0, "", false
+	}
+	d, err := r.direntAtIndex(r.hdr.mainPage)
+	if err != nil {
+		return 0, "", false
+	}
+	return d.namespace, d.url, true
+}
+
 // Get resolves the entry at (namespace, url), following one or more redirects.
 func (r *Reader) Get(namespace byte, url string) (Blob, error) {
 	target := key(namespace, url)

zim/writer.go

@@ -22,6 +22,11 @@ type Writer struct {
 	byKey      map[string]*entry
 	mainKey    string
 	noCompress bool
+	// compress turns a cluster's uncompressed data section into its stored bytes.
+	// It defaults to the built-in zstd codec; a caller can replace it with a
+	// caching compressor so a re-pack reuses the compression of unchanged
+	// clusters instead of running zstd again.
+	compress func([]byte) []byte
 }
 
 type entry struct {
@@ -47,13 +52,24 @@ func key(ns byte, url string) string { return string(ns) + url }
 
 // NewWriter returns an empty Writer.
 func NewWriter() *Writer {
-	return &Writer{byKey: map[string]*entry{}}
+	return &Writer{byKey: map[string]*entry{}, compress: zstdEncode}
 }
 
 // SetNoCompress stores every cluster uncompressed. Useful when the input is
 // already compressed or when a reader without zstd must open the file.
 func (w *Writer) SetNoCompress(v bool) { w.noCompress = v }
 
+// SetCompress replaces the cluster compressor. The function must return
+// zstd-compressed bytes (the writer marks those clusters as zstd), so a caching
+// wrapper can short-circuit unchanged clusters while still producing a valid,
+// byte-identical archive. A nil function restores the built-in codec.
+func (w *Writer) SetCompress(f func([]byte) []byte) {
+	if f == nil {
+		f = zstdEncode
+	}
+	w.compress = f
+}
+
 // AddContent adds a content entry. A later add with the same namespace and url
 // replaces the earlier one. An empty title defaults to the url.
 func (w *Writer) AddContent(namespace byte, url, title, mime string, data []byte) {
@@ -71,6 +87,16 @@ func (w *Writer) AddMetadata(name, value string) {
 	w.put(&entry{namespace: NamespaceMetadata, url: name, title: name, mime: "text/plain", data: []byte(value)})
 }
 
+// AddMetadataBytes adds an 'M' namespace entry with an explicit MIME, for binary
+// metadata such as Illustrator_48x48@1, the 48x48 PNG favicon Kiwix shows as the
+// archive's icon.
+func (w *Writer) AddMetadataBytes(name, mime string, data []byte) {
+	if mime == "" {
+		mime = "application/octet-stream"
+	}
+	w.put(&entry{namespace: NamespaceMetadata, url: name, title: name, mime: mime, data: data})
+}
+
 // AddRedirect adds a redirect from (namespace,url) to (targetNamespace,targetURL).
 func (w *Writer) AddRedirect(namespace byte, url, title string, targetNamespace byte, targetURL string) {
 	if title == "" {
@@ -175,7 +201,7 @@ func (w *Writer) buildPlan() (plan, error) {
 	clusters := w.packClusters(ents)
 	p.clusters = make([][]byte, len(clusters))
 	for i, c := range clusters {
-		p.clusters[i] = c.encode(w.noCompress)
+		p.clusters[i] = c.encode(w.noCompress, w.compress)
 	}
 
 	// 5. Directory entry bytes (URL order).
@@ -293,7 +319,10 @@ func indexOf(cs []*clusterBuf, c *clusterBuf) int {
 // encode renders a cluster: an info byte followed by the (optionally zstd)
 // data section, which is an offset table of (N+1) uint32 values then the N
 // concatenated blobs. Offsets are relative to the start of the data section.
-func (c *clusterBuf) encode(noCompress bool) []byte {
+func (c *clusterBuf) encode(noCompress bool, compress func([]byte) []byte) []byte {
+	if compress == nil {
+		compress = zstdEncode
+	}
 	n := len(c.blobs)
 	tableLen := 4 * (n + 1)
 	total := tableLen
@@ -317,7 +346,7 @@ func (c *clusterBuf) encode(noCompress bool) []byte {
 	}
 	payload := data
 	if comp == compZstd {
-		payload = zstdEncode(data)
+		payload = compress(data)
 	} else {
 		comp = compNone
 	}