feat: 1. add a crawl delay function to honor the Crawl-delay directive parsed from robots.txt during clone. (#57)

2. add --craw-delay flag to specify/override robots directive.

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
@@ -34,7 +34,7 @@ jobs:
           cache: true
       - name: install chromium (linux)
         if: matrix.os == 'ubuntu-latest'
-        uses: browser-actions/setup-chrome@v1
+        uses: browser-actions/setup-chrome@v2.1.2
         id: chrome
       - name: gofmt
         run: |
@@ -67,13 +67,13 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
           check-latest: true
           cache: true
-      - uses: golangci/golangci-lint-action@v8
+      - uses: golangci/golangci-lint-action@v9.2.1
         with:
           version: latest
 
@@ -81,7 +81,7 @@ jobs:
   govulncheck:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
@@ -97,7 +97,7 @@ jobs:
   tidy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
@@ -117,7 +117,7 @@ jobs:
   webview:
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
@@ -133,7 +133,7 @@ jobs:
   windows-gui:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod

.github/workflows/docs.yml

@@ -26,14 +26,14 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6.0.2
+      - uses: actions/checkout@v7.0.0
         with:
           submodules: true
           # Sitemap lastmod comes from the latest content commit.
           fetch-depth: 0
 
       - name: Checkout tago
-        uses: actions/checkout@v6.0.2
+        uses: actions/checkout@v7.0.0
         with:
           repository: tamnd/tago
           path: .tago-src
@@ -107,7 +107,7 @@ jobs:
       group: cloudflare-pages-kage
       cancel-in-progress: true
     steps:
-      - uses: actions/checkout@v6.0.2
+      - uses: actions/checkout@v7.0.0
         with:
           fetch-depth: 1
           sparse-checkout: scripts/

.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
     if: github.ref_type != 'tag'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
         with:
           fetch-depth: 0
       - uses: actions/setup-go@v6
@@ -35,7 +35,7 @@ jobs:
           go-version-file: go.mod
           check-latest: true
           cache: true
-      - uses: goreleaser/goreleaser-action@v6
+      - uses: goreleaser/goreleaser-action@v7.2.2
         with:
           distribution: goreleaser
           version: "~> v2"
@@ -49,8 +49,10 @@ jobs:
       contents: write # create the GitHub release
       packages: write # push the image to ghcr.io
       id-token: write # keyless cosign signing
+    env:
+      LINUX_REPO_DISPATCH_TOKEN: ${{ secrets.LINUX_REPO_DISPATCH_TOKEN }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7.0.0
         with:
           fetch-depth: 0
       - uses: actions/setup-go@v6
@@ -60,19 +62,25 @@ jobs:
           cache: true
 
       # Build and ship the linux/arm64 image from the amd64 runner.
-      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-qemu-action@v4.1.0
-      - uses: docker/setup-buildx-action@v3
+      - uses: docker/setup-buildx-action@v4.1.0
-      - uses: docker/login-action@v3
+      - uses: docker/login-action@v4.2.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       # Tools GoReleaser shells out to for signing and SBOMs.
-      - uses: sigstore/cosign-installer@v3
+      # Pin cosign to the 2.x line. cosign 3.x makes the new bundle format the
-      - uses: anchore/sbom-action/download-syft@v0
+      # default, which ignores the --output-signature/--output-certificate flags
+      # the signs block uses and aborts. Pinning keeps the .sig/.pem outputs and
+      # stops the release tool from floating to a breaking latest.
+      - uses: sigstore/cosign-installer@v4.1.2
+        with:
+          cosign-release: "v2.6.3"
+      - uses: anchore/sbom-action/download-syft@v0.24.0
 
-      - uses: goreleaser/goreleaser-action@v6
+      - uses: goreleaser/goreleaser-action@v7.2.2
         with:
           distribution: goreleaser
           version: "~> v2"
@@ -85,3 +93,20 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }}
           SCOOP_BUCKET_GITHUB_TOKEN: ${{ secrets.SCOOP_BUCKET_GITHUB_TOKEN }}
+
+      # Rebuild the Linux apt/dnf repository with the packages just released.
+      # Skipped when the dispatch token is unset, so the release never depends
+      # on it being configured.
+      - name: Refresh the Linux package repository
+        if: env.LINUX_REPO_DISPATCH_TOKEN != ''
+        run: |
+          code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
+            -H "Authorization: Bearer $LINUX_REPO_DISPATCH_TOKEN" \
+            -H "Accept: application/vnd.github+json" \
+            https://api.github.com/repos/tamnd/linux-repo/dispatches \
+            -d '{"event_type":"package-released"}' || true)
+          if [ "$code" = "204" ]; then
+            echo "Linux repo refresh triggered"
+          else
+            echo "Linux repo refresh not accepted (HTTP $code); skipping"
+          fi

.goreleaser.yaml

@@ -151,11 +151,11 @@ homebrew_casks:
     repository:
       owner: tamnd
       name: homebrew-tap
-      token: '{{ envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}'
+      token: '{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}'
     directory: Casks
     homepage: https://github.com/tamnd/kage
     description: Clone any website for offline viewing, with the JavaScript stripped out
-    skip_upload: '{{ if envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}'
+    skip_upload: '{{ if index .Env "HOMEBREW_TAP_GITHUB_TOKEN" }}false{{ else }}true{{ end }}'
     commit_author:
       name: Duc-Tam Nguyen
       email: tamnd87@gmail.com
@@ -168,11 +168,12 @@ scoops:
     repository:
       owner: tamnd
       name: scoop-bucket
-      token: '{{ envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}'
+      token: '{{ .Env.SCOOP_BUCKET_GITHUB_TOKEN }}'
+    directory: bucket
     homepage: https://github.com/tamnd/kage
     description: Clone any website for offline viewing, with the JavaScript stripped out
     license: MIT
-    skip_upload: '{{ if envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}'
+    skip_upload: '{{ if index .Env "SCOOP_BUCKET_GITHUB_TOKEN" }}false{{ else }}true{{ end }}'
     commit_author:
       name: Duc-Tam Nguyen
       email: tamnd87@gmail.com

CHANGELOG.md

@@ -6,6 +6,19 @@ All notable changes to kage are recorded here. The format follows
 
 ## [Unreleased]
 
+## [0.3.6] - 2026-06-19
+
+### Fixed
+
+- `kage clone --mobile` now produces a correct layout when the source page uses table-based navigation with image maps, as paulgraham.com does.
+  Three rendering problems appeared in Kiwix iOS after the 0.3.5 release: a tall blank box at the top of every page, content clipped on the right edge, and a narrow spacer column occupying screen space.
+  The blank box came from hiding only the `<img usemap>` element while leaving its `<td>` container in place; the cell collapsed to empty but kept its full height.
+  `td:has(>img[usemap])` now hides the entire nav column rather than just the image inside it.
+  The right-side clip came from `width="435"` set directly as an HTML attribute on the inner content table; the earlier `max-width` CSS rule does not override HTML attributes.
+  A new `[width]{width:auto!important;max-width:100%!important}` rule cancels every fixed HTML width attribute on any element — tables, cells, and images alike — so the content column stretches to fill the phone screen.
+  The spacer column (a 26 px `<td>` holding a 1×1 transparent GIF) kept its allocated width for the same reason; `td:has(>img[src*="trans_1x1"]:only-child)` hides it alongside the nav column.
+  Two additional rules round out the fix: `*{box-sizing:border-box}` prevents padding from pushing content past the viewport edge, and `img{max-width:100%;height:auto}` keeps any inline photos within their column.
+
 ## [0.3.5] - 2026-06-19
 
 ### Changed

README.md

@@ -24,7 +24,27 @@ Full docs and guides live at **[kage.tamnd.com](https://kage.tamnd.com)**.
 go install github.com/tamnd/kage/cmd/kage@latest
 ```
 
-Prefer a prebuilt binary? Grab an archive, a `.deb`/`.rpm`/`.apk`, or a checksum from [releases](https://github.com/tamnd/kage/releases). Or skip installing Chrome yourself and use the container image, which bundles Chromium:
+Prefer a prebuilt binary? Grab an archive, a `.deb`/`.rpm`/`.apk`, or a checksum from [releases](https://github.com/tamnd/kage/releases). Or let a package manager handle it:
+
+```bash
+# Homebrew (macOS)
+brew install --cask tamnd/tap/kage
+
+# Scoop (Windows)
+scoop bucket add tamnd https://github.com/tamnd/scoop-bucket
+scoop install kage
+
+# apt (Debian, Ubuntu)
+curl -fsSL https://tamnd.github.io/linux-repo/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg
+echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" | sudo tee /etc/apt/sources.list.d/tamnd.list
+sudo apt update && sudo apt install kage
+
+# dnf (Fedora, RHEL)
+sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo
+sudo dnf install kage
+```
+
+Or skip installing Chrome yourself and use the container image, which bundles Chromium:
 
 ```bash
 docker run --rm -v "$PWD/out:/out" ghcr.io/tamnd/kage clone paulgraham.com
@@ -102,6 +122,7 @@ The flags you'll actually reach for:
 | `--scroll` | `false` | Auto-scroll each page to trigger lazy loading |
 | `--workers` | `4` | How many pages to render at once |
 | `--no-robots` | `false` | Ignore `robots.txt` (be nice) |
+| `--crawl-delay` | `0s` | Override robots.txt `Crawl-delay` between page starts |
 | `-f, --force` | `false` | Delete any existing mirror for the host first |
 | `--chrome` | | Path to the Chrome/Chromium binary |
 

browser/leakless.go

@@ -0,0 +1,7 @@
+package browser
+
+import "runtime"
+
+func launcherLeakless() bool {
+	return runtime.GOOS != "windows"
+}

browser/pool.go

@@ -165,7 +165,7 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
 
 	controlURL := p.opts.ControlURL
 	if controlURL == "" {
-		l := launcher.New().
+		l := launcher.New().Leakless(launcherLeakless()).
 			Headless(p.opts.Headless).
 			Set("disable-blink-features", "AutomationControlled").
 			Set("disable-gpu", "")

browser/pool_test.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -83,6 +84,14 @@ func TestDisableSandboxContainer(t *testing.T) {
 	}
 }
 
+func TestLauncherLeaklessDisabledOnWindows(t *testing.T) {
+	got := launcherLeakless()
+	want := runtime.GOOS != "windows"
+	if got != want {
+		t.Errorf("launcherLeakless() = %v on %s; want %v", got, runtime.GOOS, want)
+	}
+}
+
 func TestRenderCapturesFinalDOM(t *testing.T) {
 	if testing.Short() {
 		t.Skip("render test drives Chrome; skipped under -short")

cli/clone.go

@@ -16,37 +16,38 @@ import (
 
 // cloneFlags holds the parsed flag values for one invocation.
 type cloneFlags struct {
-	out           string
+	out            string
-	reserved      string
+	reserved       string
-	workers       int
+	workers        int
-	assetWorkers  int
+	assetWorkers   int
-	browserPages  int
+	browserPages   int
-	maxPages      int
+	maxPages       int
-	maxDepth      int
+	maxDepth       int
-	traversal     string
+	traversal      string
-	maxAssetMB    int64
+	maxAssetMB     int64
-	keepMedia     bool
+	keepMedia      bool
-	skipExt       []string
+	skipExt        []string
-	allAssetHosts bool
+	allAssetHosts  bool
-	timeout       time.Duration
+	timeout        time.Duration
-	settle        time.Duration
+	settle         time.Duration
-	renderTO      time.Duration
+	renderTO       time.Duration
-	scroll        bool
+	scroll         bool
-	userAgent     string
+	userAgent      string
-	subdomains    bool
+	subdomains     bool
-	scopePrefix   string
+	scopePrefix    string
-	exclude       []string
+	exclude        []string
-	noRobots      bool
+	noRobots       bool
-	noSitemap     bool
+	crawlDelay     time.Duration
-	headful       bool
+	noSitemap      bool
-	keepNoscript    bool
+	headful        bool
-	mobileReadable  bool
+	keepNoscript   bool
-	chromeBin       string
+	mobileReadable bool
-	controlURL    string
+	chromeBin      string
-	noResume      bool
+	controlURL     string
-	refresh       bool
+	noResume       bool
-	force         bool
+	refresh        bool
-	quiet         bool
+	force          bool
+	quiet          bool
 }
 
 func newCloneCmd() *cobra.Command {
@@ -84,6 +85,7 @@ func newCloneCmd() *cobra.Command {
 	fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix")
 	fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)")
 	fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)")
+	fs.DurationVar(&f.crawlDelay, "crawl-delay", 0, "override robots.txt Crawl-delay between page starts (0 = use robots.txt)")
 	fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml")
 	fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)")
 	fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap <noscript> content instead of dropping it")
@@ -102,6 +104,9 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
 	if err != nil {
 		return fmt.Errorf("invalid url %q: %w", arg, err)
 	}
+	if f.crawlDelay < 0 {
+		return fmt.Errorf("--crawl-delay must be >= 0")
+	}
 
 	cfg := clone.DefaultConfig()
 	cfg.OutDir = f.out
@@ -139,6 +144,7 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
 	cfg.ScopePrefix = f.scopePrefix
 	cfg.ExcludePaths = f.exclude
 	cfg.RespectRobots = !f.noRobots
+	cfg.CrawlDelay = f.crawlDelay
 	cfg.FollowSitemap = !f.noSitemap
 	cfg.Headless = !f.headful
 	cfg.KeepNoscript = f.keepNoscript

clone/cloner.go

@@ -19,6 +19,7 @@ import (
 	"github.com/tamnd/kage/sanitize"
 	"github.com/tamnd/kage/urlx"
 	"golang.org/x/net/html"
+	"golang.org/x/time/rate"
 )
 
 // Logf is an optional sink for human-readable progress lines.
@@ -43,9 +44,12 @@ type Cloner struct {
 	mu         sync.Mutex
 	seenAssets map[string]bool
 	enqueued   int // pages offered to the queue
-	wg         sync.WaitGroup
+
-	pageJobs   chan pageItem
+	crawlLimiter *rate.Limiter
-	assetJobs  chan assetItem
+
+	wg        sync.WaitGroup
+	pageJobs  chan pageItem
+	assetJobs chan assetItem
 
 	muContent   sync.Mutex
 	seenContent map[string]string // sha-256 of page bytes -> first path written
@@ -144,6 +148,7 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) {
 	defer func() { _ = c.pool.Close() }()
 
 	c.loadRobots(ctx)
+	c.setupCrawlDelayLimiter()
 
 	// Start workers.
 	var workers sync.WaitGroup
@@ -218,6 +223,19 @@ func (c *Cloner) loadRobots(ctx context.Context) {
 	c.robots = robots.Parse(string(data), "kage")
 }
 
+func (c *Cloner) setupCrawlDelayLimiter() {
+	delay := c.cfg.CrawlDelay
+	if delay <= 0 && c.cfg.RespectRobots && c.robots != nil {
+		delay = c.robots.CrawlDelay
+	}
+	if delay <= 0 {
+		c.crawlLimiter = nil
+		return
+	}
+
+	c.crawlLimiter = rate.NewLimiter(rate.Every(delay), 1)
+}
+
 // seedSitemaps adds in-scope sitemap URLs (from robots and the default path) to
 // the frontier.
 func (c *Cloner) seedSitemaps(ctx context.Context) {
@@ -252,6 +270,9 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 		c.stats.skipped.Add(1)
 		return
 	}
+	if !c.waitForCrawlDelay(ctx) {
+		return
+	}
 
 	res, err := c.pool.Render(ctx, j.u.String())
 	if err != nil {
@@ -324,6 +345,15 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 	c.stats.recordPage(c.pagePathKey(j.u), deduped)
 }
 
+// waitForCrawlDelay spaces page render starts.
+func (c *Cloner) waitForCrawlDelay(ctx context.Context) bool {
+	if c.crawlLimiter == nil {
+		return true
+	}
+
+	return c.crawlLimiter.Wait(ctx) == nil
+}
+
 // processAsset downloads one asset, rewriting CSS references on the way, and
 // writes it to its deterministic local path.
 func (c *Cloner) processAsset(ctx context.Context, j assetItem) {

clone/cloner_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/tamnd/kage/browser"
+	"github.com/tamnd/kage/robots"
 	"github.com/tamnd/kage/urlx"
 )
 
@@ -178,6 +179,56 @@ func TestPageKeyCollapsesDuplicates(t *testing.T) {
 	}
 }
 
+func TestCrawlDelaySpacesPageStarts(t *testing.T) {
+	seed, _ := urlx.ParseSeed("https://ex.com")
+	cfg := DefaultConfig()
+	cfg.RespectRobots = true
+	c := New(seed, cfg, nil)
+	c.robots = &robots.Matcher{CrawlDelay: 20 * time.Millisecond}
+	c.setupCrawlDelayLimiter()
+
+	ctx := context.Background()
+	if !c.waitForCrawlDelay(ctx) {
+		t.Fatal("first crawl-delay wait returned false")
+	}
+
+	start := time.Now()
+	if !c.waitForCrawlDelay(ctx) {
+		t.Fatal("second crawl-delay wait returned false")
+	}
+	if elapsed := time.Since(start); elapsed < 15*time.Millisecond {
+		t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed)
+	}
+}
+
+func TestCrawlDelayFlagOverridesRobots(t *testing.T) {
+	seed, _ := urlx.ParseSeed("https://ex.com")
+	cfg := DefaultConfig()
+	cfg.RespectRobots = true
+	cfg.CrawlDelay = 20 * time.Millisecond
+	c := New(seed, cfg, nil)
+	c.robots = &robots.Matcher{CrawlDelay: time.Minute}
+	c.setupCrawlDelayLimiter()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+	defer cancel()
+	if !c.waitForCrawlDelay(ctx) {
+		t.Fatal("first crawl-delay wait returned false")
+	}
+
+	start := time.Now()
+	if !c.waitForCrawlDelay(ctx) {
+		t.Fatal("second crawl-delay wait returned false")
+	}
+	elapsed := time.Since(start)
+	if elapsed < 15*time.Millisecond {
+		t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed)
+	}
+	if elapsed > 150*time.Millisecond {
+		t.Fatalf("second crawl-delay wait = %v, override likely ignored", elapsed)
+	}
+}
+
 func mustURL(t *testing.T, raw string) *url.URL {
 	t.Helper()
 	u, err := url.Parse(raw)

clone/config.go

@@ -57,6 +57,7 @@ type Config struct {
 	ExcludePaths      []string
 
 	RespectRobots  bool
+	CrawlDelay     time.Duration // override robots.txt Crawl-delay when > 0
 	FollowSitemap  bool
 	Headless       bool
 	KeepNoscript   bool

docs/content/getting-started/installation.md

@@ -1,6 +1,6 @@
 ---
 title: "Installation"
-description: "Install kage from Go, Homebrew, a release archive, a Linux package, or the container image, and point it at a browser."
+description: "Install kage from Go, Homebrew, Scoop, a release archive, a Linux package, or the container image, and point it at a browser."
 weight: 20
 ---
 
@@ -12,10 +12,38 @@ kage is a single binary. Pick whichever channel suits you.
 go install github.com/tamnd/kage/cmd/kage@latest
 ```
 
-## Homebrew
+## Homebrew (macOS)
 
 ```bash
-brew install tamnd/tap/kage
+brew install --cask tamnd/tap/kage
+```
+
+The cask installs the prebuilt macOS binary. On Linux, use the packages below or
+`go install`.
+
+## Scoop (Windows)
+
+```bash
+scoop bucket add tamnd https://github.com/tamnd/scoop-bucket
+scoop install kage
+```
+
+## Linux (apt and dnf)
+
+A signed apt and dnf repository tracks every release, so `apt upgrade` and
+`dnf upgrade` keep kage current.
+
+```bash
+# Debian, Ubuntu
+curl -fsSL https://tamnd.github.io/linux-repo/gpg.key \
+  | sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg
+echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" \
+  | sudo tee /etc/apt/sources.list.d/tamnd.list
+sudo apt update && sudo apt install kage
+
+# Fedora, RHEL
+sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo
+sudo dnf install kage
 ```
 
 ## Release archives and Linux packages
@@ -24,14 +52,14 @@ Every [release](https://github.com/tamnd/kage/releases) attaches `tar.gz`
 archives (and a `.zip` for Windows) for Linux, macOS, Windows, and FreeBSD, plus
 `.deb`, `.rpm`, and `.apk` packages and a `checksums.txt` with a cosign
 signature. Download the one for your platform, extract `kage`, and put it on your
-`PATH`.
+`PATH`. To install a package directly without the repo above:
 
 ```bash
 # Debian/Ubuntu
-sudo dpkg -i kage_*_linux_amd64.deb
+sudo dpkg -i kage_*_amd64.deb
 
 # Fedora/RHEL
-sudo rpm -i kage_*_linux_amd64.rpm
+sudo rpm -i kage-*.x86_64.rpm
 ```
 
 ## Container

docs/content/reference/cli.md

@@ -47,6 +47,7 @@ images, and fonts, and writes a browsable mirror to `<out>/<host>/`.
 | Flag | Default | Meaning |
 |------|---------|---------|
 | `--no-robots` | `false` | Ignore `robots.txt` |
+| `--crawl-delay` | `0s` | Override robots.txt `Crawl-delay` between page starts (0 = use robots.txt) |
 | `--no-sitemap` | `false` | Do not seed URLs from `sitemap.xml` |
 | `--user-agent` | Chrome UA | User-Agent for asset and robots fetches |
 

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6
 	golang.org/x/image v0.42.0
 	golang.org/x/net v0.56.0
+	golang.org/x/time v0.15.0
 )
 
 require (

go.sum

@@ -123,6 +123,8 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
 golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
 golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=

sanitize/sanitize.go

@@ -303,21 +303,37 @@ func findElement(n *html.Node, a atom.Atom) *html.Node {
 	return nil
 }
 
-// mobileCSS is injected when MobileReadable is set. It targets legacy "font
+// mobileCSS is injected when MobileReadable is set. It rewrites font-era
-// era" HTML that renders as microscopic text on mobile:
+// HTML for comfortable reading on a phone. Key rules:
-//   - :root font-size 18 px — baseline all em/rem sizes upward
+//
-//   - body — centre, cap width, add padding, loosen line height
+//   - box-sizing:border-box — makes padding predictable in a layout built with
-//   - font element — override the in-HTML size/face attributes that sites like
+//     HTML width attributes, so our padding doesn't cause overflow.
-//     paulgraham.com embed directly in the markup (e.g. <font size="2">)
+//   - body — no fixed max-width here; let the table rules handle width instead.
-//   - table/td — prevent overflow; add minimal cell breathing room
+//     overflow-x:hidden catches any stray overflow without a scrollbar.
-//   - img[usemap], map — image-map navigation is useless offline (the image
+//   - font element — overrides in-HTML size/face attributes (e.g. <font size="2">).
-//     itself usually 404s from an external CDN); hide both the image and the map
+//   - [width],[height] — cancels all HTML attribute widths/heights on every
-const mobileCSS = `body{max-width:720px;margin:0 auto;padding:.75em 1em;line-height:1.7;font-family:Georgia,"Times New Roman",serif}` +
+//     element (tables, tds, imgs, etc.) so fixed-pixel columns become fluid.
+//   - table — fluid, auto layout, no horizontal scroll.
+//   - td — auto width so a three-column table (nav | spacer | content) collapses
+//     to one usable column once the nav td is hidden.
+//   - img — responsive: never wider than its container.
+//   - td:has(>img[usemap]) — hides the entire nav column td, not just the image
+//     inside it; hiding only the img left a tall empty white box.
+//   - td:has(>img[src*="trans_1x1"]) — hides the 26 px spacer column td (a 1×1
+//     transparent GIF whose only job was spacing in the original table layout).
+const mobileCSS = `*{box-sizing:border-box}` +
 	`:root{font-size:18px}` +
+	`body{margin:0;padding:.75em 1em;line-height:1.7;font-family:Georgia,"Times New Roman",serif;overflow-x:hidden}` +
 	`font{font-size:1rem!important;font-family:inherit!important;color:inherit!important}` +
-	`table{max-width:100%!important;word-break:break-word}` +
+	`[width]{width:auto!important;max-width:100%!important}` +
-	`td,th{padding:.25em!important}` +
+	`[height]{height:auto!important}` +
-	`img[usemap],map{display:none!important}`
+	`table{width:100%!important;max-width:100%!important;table-layout:auto!important;border-collapse:collapse!important;word-break:break-word}` +
+	`td,th{width:auto!important;max-width:100%!important;padding:.35em .5em!important;vertical-align:top!important;overflow-wrap:break-word}` +
+	`img{max-width:100%!important;height:auto!important}` +
+	`img[usemap],map{display:none!important}` +
+	`td:has(>img[usemap]),td:has(>map){display:none!important}` +
+	`img[src*="trans_1x1"],img[src*="spacer"],img[height="1"],img[width="1"]{display:none!important}` +
+	`td:has(>img[src*="trans_1x1"]:only-child),td:has(>img[height="1"]:only-child){display:none!important}`
 
 // ensureViewport inserts <meta name="viewport" content="width=device-width,
 // initial-scale=1"> at the top of <head> when the document does not already

sanitize/sanitize_test.go

@@ -218,6 +218,40 @@ func TestMobileReadableInjectsViewportAndCSS(t *testing.T) {
 	if !strings.Contains(s, "font{font-size:1rem") {
 		t.Error("font override not in mobile CSS")
 	}
+	// Fluid table rules must be present.
+	if !strings.Contains(s, "table{width:100%") {
+		t.Error("fluid table rule missing")
+	}
+	// [width] override must be present so fixed HTML width attributes are cancelled.
+	if !strings.Contains(s, "[width]{width:auto") {
+		t.Error("[width] override missing")
+	}
+}
+
+func TestMobileReadableHidesNavColumnTd(t *testing.T) {
+	// paulgraham.com wraps its image-map nav in a <td>. Hiding only the img
+	// leaves a tall empty box; the CSS must also target the containing td.
+	in := `<html><head><title>T</title></head><body>` +
+		`<table><tr>` +
+		`<td><map name="nav"><area shape="rect" coords="0,0,67,21" href="index.html"></map>` +
+		`<img src="nav.gif" width="69" height="357" usemap="#nav"/></td>` +
+		`<td><img src="https://cdn.example.com/trans_1x1.gif" height="1" width="26"/></td>` +
+		`<td><p>Essay text.</p></td>` +
+		`</tr></table>` +
+		`</body></html>`
+	out, _, err := Strip([]byte(in), Options{MobileReadable: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := string(out)
+	// td:has(>img[usemap]) rule must be present so the whole nav column is hidden.
+	if !strings.Contains(s, "td:has(>img[usemap])") {
+		t.Error("td:has(>img[usemap]) rule missing from mobile CSS")
+	}
+	// td:has(>img[src*="trans_1x1"]) rule must be present for spacer column.
+	if !strings.Contains(s, `td:has(>img[src*="trans_1x1"]`) {
+		t.Error(`td:has(>img[src*="trans_1x1"]) spacer-column rule missing from mobile CSS`)
+	}
 }
 
 func TestMobileReadableSkipsExistingViewport(t *testing.T) {