Compare commits
27 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 35860d5aaa | |||
| 7483efd2a9 | |||
| 320b21a2ba | |||
| 2dabb93a78 | |||
| 8bc9c8bea1 | |||
| 28d08ade88 | |||
| 1e59c10e6f | |||
| 105e68f036 | |||
| 792878d6f3 | |||
| 910619bc7a | |||
| e0e4096f93 | |||
| c176719031 | |||
| a1b82ebd1b | |||
| 5ad8fff9df | |||
| 7157144c75 | |||
| de1c0adde3 | |||
| 16bbd1b9d0 | |||
| 3220f6d230 | |||
| 9d452ddf11 | |||
| c8fc6faaa3 | |||
| c5b39d6816 | |||
| 057937938d | |||
| 4e920c57ec | |||
| 2c15f79ff5 | |||
| 060b4b6449 | |||
| 3117c4c55c | |||
| 6655a434b9 |
@@ -26,7 +26,7 @@ jobs:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
cache: true
|
||||
- name: install chromium (linux)
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
uses: browser-actions/setup-chrome@v1
|
||||
uses: browser-actions/setup-chrome@v2.1.2
|
||||
id: chrome
|
||||
- name: gofmt
|
||||
run: |
|
||||
@@ -67,13 +67,13 @@ jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: true
|
||||
- uses: golangci/golangci-lint-action@v8
|
||||
- uses: golangci/golangci-lint-action@v9.2.1
|
||||
with:
|
||||
version: latest
|
||||
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
govulncheck:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
tidy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
@@ -117,7 +117,7 @@ jobs:
|
||||
webview:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
@@ -133,7 +133,7 @@ jobs:
|
||||
windows-gui:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -26,14 +26,14 @@ jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6.0.2
|
||||
- uses: actions/checkout@v7.0.0
|
||||
with:
|
||||
submodules: true
|
||||
# Sitemap lastmod comes from the latest content commit.
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Checkout tago
|
||||
uses: actions/checkout@v6.0.2
|
||||
uses: actions/checkout@v7.0.0
|
||||
with:
|
||||
repository: tamnd/tago
|
||||
path: .tago-src
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
group: cloudflare-pages-kage
|
||||
cancel-in-progress: true
|
||||
steps:
|
||||
- uses: actions/checkout@v6.0.2
|
||||
- uses: actions/checkout@v7.0.0
|
||||
with:
|
||||
fetch-depth: 1
|
||||
sparse-checkout: scripts/
|
||||
|
||||
@@ -27,15 +27,15 @@ jobs:
|
||||
if: github.ref_type != 'tag'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-go@v6
|
||||
- uses: actions/setup-go@v6.4.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: true
|
||||
- uses: goreleaser/goreleaser-action@v6
|
||||
- uses: goreleaser/goreleaser-action@v7.2.2
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: "~> v2"
|
||||
@@ -49,30 +49,38 @@ jobs:
|
||||
contents: write # create the GitHub release
|
||||
packages: write # push the image to ghcr.io
|
||||
id-token: write # keyless cosign signing
|
||||
env:
|
||||
LINUX_REPO_DISPATCH_TOKEN: ${{ secrets.LINUX_REPO_DISPATCH_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-go@v6
|
||||
- uses: actions/setup-go@v6.4.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: true
|
||||
|
||||
# Build and ship the linux/arm64 image from the amd64 runner.
|
||||
- uses: docker/setup-qemu-action@v3
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
- uses: docker/setup-qemu-action@v4.1.0
|
||||
- uses: docker/setup-buildx-action@v4.1.0
|
||||
- uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Tools GoReleaser shells out to for signing and SBOMs.
|
||||
- uses: sigstore/cosign-installer@v3
|
||||
- uses: anchore/sbom-action/download-syft@v0
|
||||
# Pin cosign to the 2.x line. cosign 3.x makes the new bundle format the
|
||||
# default, which ignores the --output-signature/--output-certificate flags
|
||||
# the signs block uses and aborts. Pinning keeps the .sig/.pem outputs and
|
||||
# stops the release tool from floating to a breaking latest.
|
||||
- uses: sigstore/cosign-installer@v4.1.2
|
||||
with:
|
||||
cosign-release: "v2.6.3"
|
||||
- uses: anchore/sbom-action/download-syft@v0.24.0
|
||||
|
||||
- uses: goreleaser/goreleaser-action@v6
|
||||
- uses: goreleaser/goreleaser-action@v7.2.2
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: "~> v2"
|
||||
@@ -85,3 +93,20 @@ jobs:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }}
|
||||
SCOOP_BUCKET_GITHUB_TOKEN: ${{ secrets.SCOOP_BUCKET_GITHUB_TOKEN }}
|
||||
|
||||
# Rebuild the Linux apt/dnf repository with the packages just released.
|
||||
# Skipped when the dispatch token is unset, so the release never depends
|
||||
# on it being configured.
|
||||
- name: Refresh the Linux package repository
|
||||
if: env.LINUX_REPO_DISPATCH_TOKEN != ''
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "Authorization: Bearer $LINUX_REPO_DISPATCH_TOKEN" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
https://api.github.com/repos/tamnd/linux-repo/dispatches \
|
||||
-d '{"event_type":"package-released"}' || true)
|
||||
if [ "$code" = "204" ]; then
|
||||
echo "Linux repo refresh triggered"
|
||||
else
|
||||
echo "Linux repo refresh not accepted (HTTP $code); skipping"
|
||||
fi
|
||||
|
||||
+14
-4
@@ -151,14 +151,23 @@ homebrew_casks:
|
||||
repository:
|
||||
owner: tamnd
|
||||
name: homebrew-tap
|
||||
token: '{{ envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}'
|
||||
token: '{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}'
|
||||
directory: Casks
|
||||
homepage: https://github.com/tamnd/kage
|
||||
description: Clone any website for offline viewing, with the JavaScript stripped out
|
||||
skip_upload: '{{ if envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}'
|
||||
skip_upload: '{{ if index .Env "HOMEBREW_TAP_GITHUB_TOKEN" }}false{{ else }}true{{ end }}'
|
||||
commit_author:
|
||||
name: Duc-Tam Nguyen
|
||||
email: tamnd87@gmail.com
|
||||
# Homebrew quarantines cask artifacts, and Gatekeeper kills quarantined
|
||||
# binaries that are only ad-hoc signed (which cross-compiled Go binaries
|
||||
# are). Strip the attribute at install so the binary runs first try.
|
||||
hooks:
|
||||
post:
|
||||
install: |
|
||||
if system_command("/usr/bin/xattr", args: ["-h"]).exit_status.zero?
|
||||
system_command "/usr/bin/xattr", args: ["-dr", "com.apple.quarantine", "#{staged_path}/kage"]
|
||||
end
|
||||
|
||||
scoops:
|
||||
# Scoop manifest for Windows, pushed to the bucket repository. It installs the
|
||||
@@ -168,11 +177,12 @@ scoops:
|
||||
repository:
|
||||
owner: tamnd
|
||||
name: scoop-bucket
|
||||
token: '{{ envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}'
|
||||
token: '{{ .Env.SCOOP_BUCKET_GITHUB_TOKEN }}'
|
||||
directory: bucket
|
||||
homepage: https://github.com/tamnd/kage
|
||||
description: Clone any website for offline viewing, with the JavaScript stripped out
|
||||
license: MIT
|
||||
skip_upload: '{{ if envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}'
|
||||
skip_upload: '{{ if index .Env "SCOOP_BUCKET_GITHUB_TOKEN" }}false{{ else }}true{{ end }}'
|
||||
commit_author:
|
||||
name: Duc-Tam Nguyen
|
||||
email: tamnd87@gmail.com
|
||||
|
||||
+42
-1
@@ -6,6 +6,46 @@ All notable changes to kage are recorded here. The format follows
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.3.9] - 2026-07-08
|
||||
|
||||
### Fixed
|
||||
|
||||
- The Windows build no longer embeds the leakless watchdog binary that Windows Defender flags as `Trojan:Win32/Kepavll!rfn`, which made a fresh `scoop install` fail with a virus warning on `leakless.exe` ([#68](https://github.com/tamnd/kage/issues/68)).
|
||||
go-rod's launcher imports [leakless](https://github.com/ysmood/leakless), which base64/gzip-embeds a prebuilt helper for every platform and links the Windows one into `kage.exe`.
|
||||
kage already launches Chrome with leakless disabled, so the helper never ran, only added the flagged bytes.
|
||||
A `replace` directive now points the package at an API-compatible stub under `third_party/leakless` that carries no embedded binary, dropping about 1.28 MB from the Windows build.
|
||||
|
||||
## [0.3.6] - 2026-06-19
|
||||
|
||||
### Fixed
|
||||
|
||||
- `kage clone --mobile` now produces a correct layout when the source page uses table-based navigation with image maps, as paulgraham.com does.
|
||||
Three rendering problems appeared in Kiwix iOS after the 0.3.5 release: a tall blank box at the top of every page, content clipped on the right edge, and a narrow spacer column occupying screen space.
|
||||
The blank box came from hiding only the `<img usemap>` element while leaving its `<td>` container in place; the cell collapsed to empty but kept its full height.
|
||||
`td:has(>img[usemap])` now hides the entire nav column rather than just the image inside it.
|
||||
The right-side clip came from `width="435"` set directly as an HTML attribute on the inner content table; the earlier `max-width` CSS rule does not override HTML attributes.
|
||||
A new `[width]{width:auto!important;max-width:100%!important}` rule cancels every fixed HTML width attribute on any element — tables, cells, and images alike — so the content column stretches to fill the phone screen.
|
||||
The spacer column (a 26 px `<td>` holding a 1×1 transparent GIF) kept its allocated width for the same reason; `td:has(>img[src*="trans_1x1"]:only-child)` hides it alongside the nav column.
|
||||
Two additional rules round out the fix: `*{box-sizing:border-box}` prevents padding from pushing content past the viewport edge, and `img{max-width:100%;height:auto}` keeps any inline photos within their column.
|
||||
|
||||
## [0.3.5] - 2026-06-19
|
||||
|
||||
### Changed
|
||||
|
||||
- Each saved page is now stored in a packed ZIM under its own `<title>` instead of its URL path, so a ZIM reader's search box suggests pages by their readable title.
|
||||
Typing into Kiwix's search now offers "Five Founders" or "Female Founders" rather than a filename, because the title pointer list a reader's suggestion search walks carries the real page titles.
|
||||
A page with no `<title>` still falls back to its path, and the per-page title also flows into the `title` column of `kage parquet export`.
|
||||
|
||||
### Added
|
||||
|
||||
- `kage clone --mobile` makes legacy "font-era" sites readable on a phone.
|
||||
Sites from the 1990s and early 2000s — paulgraham.com is a good example — embed typography directly in the HTML with `<font size="2" face="verdana">`, table-based layouts, and no viewport declaration.
|
||||
A mobile browser receiving that markup without a viewport meta shrinks everything to desktop scale, and the `<font size="2">` instruction then makes the already-small text microscopic.
|
||||
Passing `--mobile` injects two things into every saved page before it is written: a `<meta name="viewport" content="width=device-width, initial-scale=1">` tag so the browser stops shrinking, and a small `<style>` block that lifts the base font size to 18 px, inherits that size through `<font>` elements, caps the content width at 720 px, loosens line height to 1.7, and hides image-map navigation elements (usually a GIF served from an external CDN that 404s offline anyway).
|
||||
The override is deliberately last in `<head>` so it wins specificity ties, and it does not touch pages that already carry a viewport and readable type sizes.
|
||||
- The packing guide now documents how search works on a kage archive: title suggestions in any ZIM reader, and full-text search of page bodies through `kage parquet export` and DuckDB.
|
||||
A Xapian full-text index is deliberately not written, since Xapian is GPL and kage is MIT.
|
||||
|
||||
## [0.3.4] - 2026-06-17
|
||||
|
||||
### Fixed
|
||||
@@ -224,7 +264,8 @@ can browse offline, with every script stripped out.
|
||||
a multi-arch container image on GHCR (Chromium bundled), checksums, SBOMs, and
|
||||
a cosign signature, all cut from one version tag by GoReleaser.
|
||||
|
||||
[Unreleased]: https://github.com/tamnd/kage/compare/v0.3.4...HEAD
|
||||
[Unreleased]: https://github.com/tamnd/kage/compare/v0.3.9...HEAD
|
||||
[0.3.9]: https://github.com/tamnd/kage/compare/v0.3.8...v0.3.9
|
||||
[0.3.4]: https://github.com/tamnd/kage/compare/v0.3.3...v0.3.4
|
||||
[0.3.3]: https://github.com/tamnd/kage/compare/v0.3.2...v0.3.3
|
||||
[0.3.2]: https://github.com/tamnd/kage/compare/v0.3.1...v0.3.2
|
||||
|
||||
+13
-9
@@ -17,24 +17,28 @@ ARG TARGETPLATFORM
|
||||
# chromium for rendering; ca-certificates for HTTPS; tzdata for sane timestamps;
|
||||
# the font package so rendered pages have glyphs to lay out.
|
||||
RUN apk add --no-cache chromium ca-certificates tzdata font-noto \
|
||||
&& adduser -D -H -u 10001 kage \
|
||||
&& mkdir -p /out \
|
||||
&& chown kage:kage /out
|
||||
&& mkdir -p /out
|
||||
|
||||
COPY $TARGETPLATFORM/kage /usr/bin/kage
|
||||
|
||||
USER kage
|
||||
WORKDIR /out
|
||||
|
||||
# Point kage at the bundled Chromium and write mirrors under /out by default:
|
||||
#
|
||||
# docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com
|
||||
#
|
||||
# The kage user has no home directory of its own, so HOME points at the mounted
|
||||
# /out volume. That keeps two things writable: kage's default output and resume
|
||||
# state (it lands under $HOME/data/kage), and Chrome's profile and crash
|
||||
# database. Without this both fail with a permission error in the container
|
||||
# (issue #7), and the mounted volume captures nothing.
|
||||
# The container runs as root, and that is deliberate (issue #7). A bind-mounted
|
||||
# /out is owned by whoever created it on the host, so only root can reliably
|
||||
# write into it; a fixed non-root uid cannot, and both kage's output and resume
|
||||
# state (under $HOME/data/kage) then fail with "mkdir /out: permission denied".
|
||||
# The same unwritable HOME also breaks Chrome: it launches chrome_crashpad_handler
|
||||
# with an empty crash database path, which aborts the whole browser with
|
||||
# "chrome_crashpad_handler: --database is required" and fails every render.
|
||||
# Running as root keeps /out and HOME writable whatever the host owns, so the
|
||||
# one-liner above just works. This costs nothing in the sandbox: Chrome's sandbox
|
||||
# is already off inside any container (kage drops it on container detection), so
|
||||
# root here does not loosen a boundary that was holding. HOME points at /out so
|
||||
# the default output and Chrome's writable state both land in the mounted volume.
|
||||
ENV KAGE_CHROME=/usr/bin/chromium-browser \
|
||||
HOME=/out
|
||||
|
||||
|
||||
@@ -24,7 +24,27 @@ Full docs and guides live at **[kage.tamnd.com](https://kage.tamnd.com)**.
|
||||
go install github.com/tamnd/kage/cmd/kage@latest
|
||||
```
|
||||
|
||||
Prefer a prebuilt binary? Grab an archive, a `.deb`/`.rpm`/`.apk`, or a checksum from [releases](https://github.com/tamnd/kage/releases). Or skip installing Chrome yourself and use the container image, which bundles Chromium:
|
||||
Prefer a prebuilt binary? Grab an archive, a `.deb`/`.rpm`/`.apk`, or a checksum from [releases](https://github.com/tamnd/kage/releases). Or let a package manager handle it:
|
||||
|
||||
```bash
|
||||
# Homebrew (macOS)
|
||||
brew install --cask tamnd/tap/kage
|
||||
|
||||
# Scoop (Windows)
|
||||
scoop bucket add tamnd https://github.com/tamnd/scoop-bucket
|
||||
scoop install kage
|
||||
|
||||
# apt (Debian, Ubuntu)
|
||||
curl -fsSL https://tamnd.github.io/linux-repo/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" | sudo tee /etc/apt/sources.list.d/tamnd.list
|
||||
sudo apt update && sudo apt install kage
|
||||
|
||||
# dnf (Fedora, RHEL)
|
||||
sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo
|
||||
sudo dnf install kage
|
||||
```
|
||||
|
||||
Or skip installing Chrome yourself and use the container image, which bundles Chromium:
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD/out:/out" ghcr.io/tamnd/kage clone paulgraham.com
|
||||
@@ -102,6 +122,7 @@ The flags you'll actually reach for:
|
||||
| `--scroll` | `false` | Auto-scroll each page to trigger lazy loading |
|
||||
| `--workers` | `4` | How many pages to render at once |
|
||||
| `--no-robots` | `false` | Ignore `robots.txt` (be nice) |
|
||||
| `--crawl-delay` | `0s` | Override robots.txt `Crawl-delay` between page starts |
|
||||
| `-f, --force` | `false` | Delete any existing mirror for the host first |
|
||||
| `--chrome` | | Path to the Chrome/Chromium binary |
|
||||
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
package browser
|
||||
|
||||
import "runtime"
|
||||
|
||||
func launcherLeakless() bool {
|
||||
return runtime.GOOS != "windows"
|
||||
}
|
||||
+8
-7
@@ -165,7 +165,7 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
|
||||
|
||||
controlURL := p.opts.ControlURL
|
||||
if controlURL == "" {
|
||||
l := launcher.New().
|
||||
l := launcher.New().Leakless(launcherLeakless()).
|
||||
Headless(p.opts.Headless).
|
||||
Set("disable-blink-features", "AutomationControlled").
|
||||
Set("disable-gpu", "")
|
||||
@@ -183,13 +183,14 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
|
||||
// In a container, the default /dev/shm is only 64 MB, too small for
|
||||
// Chrome's renderer on large pages, so steer it to a temp file instead.
|
||||
// Outside a container /dev/shm is roomy and faster, so leave it alone.
|
||||
// Chrome's crashpad handler also aborts with "--database is required" in a
|
||||
// minimal container, which fails the whole launch (issue #7), so turn the
|
||||
// crash reporter off there. kage never uploads Chrome crash dumps anyway.
|
||||
//
|
||||
// The "chrome_crashpad_handler: --database is required" abort seen in
|
||||
// containers (issue #7) is not fixed here: the crash-reporter flags do not
|
||||
// stop Chrome from spawning the handler. Its real cause is an unwritable
|
||||
// HOME, which leaves the crash database path empty; the image keeps HOME
|
||||
// writable instead (see the Dockerfile).
|
||||
if inContainer() {
|
||||
l = l.Set("disable-dev-shm-usage", "").
|
||||
Set("disable-crash-reporter", "").
|
||||
Set("disable-breakpad", "")
|
||||
l = l.Set("disable-dev-shm-usage", "")
|
||||
}
|
||||
|
||||
if bin := p.chromeBin(); bin != "" {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -83,6 +84,14 @@ func TestDisableSandboxContainer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLauncherLeaklessDisabledOnWindows(t *testing.T) {
|
||||
got := launcherLeakless()
|
||||
want := runtime.GOOS != "windows"
|
||||
if got != want {
|
||||
t.Errorf("launcherLeakless() = %v on %s; want %v", got, runtime.GOOS, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCapturesFinalDOM(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("render test drives Chrome; skipped under -short")
|
||||
|
||||
+39
-30
@@ -16,36 +16,38 @@ import (
|
||||
|
||||
// cloneFlags holds the parsed flag values for one invocation.
|
||||
type cloneFlags struct {
|
||||
out string
|
||||
reserved string
|
||||
workers int
|
||||
assetWorkers int
|
||||
browserPages int
|
||||
maxPages int
|
||||
maxDepth int
|
||||
traversal string
|
||||
maxAssetMB int64
|
||||
keepMedia bool
|
||||
skipExt []string
|
||||
allAssetHosts bool
|
||||
timeout time.Duration
|
||||
settle time.Duration
|
||||
renderTO time.Duration
|
||||
scroll bool
|
||||
userAgent string
|
||||
subdomains bool
|
||||
scopePrefix string
|
||||
exclude []string
|
||||
noRobots bool
|
||||
noSitemap bool
|
||||
headful bool
|
||||
keepNoscript bool
|
||||
chromeBin string
|
||||
controlURL string
|
||||
noResume bool
|
||||
refresh bool
|
||||
force bool
|
||||
quiet bool
|
||||
out string
|
||||
reserved string
|
||||
workers int
|
||||
assetWorkers int
|
||||
browserPages int
|
||||
maxPages int
|
||||
maxDepth int
|
||||
traversal string
|
||||
maxAssetMB int64
|
||||
keepMedia bool
|
||||
skipExt []string
|
||||
allAssetHosts bool
|
||||
timeout time.Duration
|
||||
settle time.Duration
|
||||
renderTO time.Duration
|
||||
scroll bool
|
||||
userAgent string
|
||||
subdomains bool
|
||||
scopePrefix string
|
||||
exclude []string
|
||||
noRobots bool
|
||||
crawlDelay time.Duration
|
||||
noSitemap bool
|
||||
headful bool
|
||||
keepNoscript bool
|
||||
mobileReadable bool
|
||||
chromeBin string
|
||||
controlURL string
|
||||
noResume bool
|
||||
refresh bool
|
||||
force bool
|
||||
quiet bool
|
||||
}
|
||||
|
||||
func newCloneCmd() *cobra.Command {
|
||||
@@ -83,9 +85,11 @@ func newCloneCmd() *cobra.Command {
|
||||
fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix")
|
||||
fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)")
|
||||
fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)")
|
||||
fs.DurationVar(&f.crawlDelay, "crawl-delay", 0, "override robots.txt Crawl-delay between page starts (0 = use robots.txt)")
|
||||
fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml")
|
||||
fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)")
|
||||
fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap <noscript> content instead of dropping it")
|
||||
fs.BoolVar(&f.mobileReadable, "mobile", false, "inject viewport and CSS overrides so legacy sites read comfortably on a phone")
|
||||
fs.StringVar(&f.chromeBin, "chrome", "", "path to the Chrome/Chromium binary")
|
||||
fs.StringVar(&f.controlURL, "control-url", "", "attach to an existing Chrome DevTools endpoint")
|
||||
fs.BoolVar(&f.noResume, "no-resume", false, "do not reuse or write resume state")
|
||||
@@ -100,6 +104,9 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid url %q: %w", arg, err)
|
||||
}
|
||||
if f.crawlDelay < 0 {
|
||||
return fmt.Errorf("--crawl-delay must be >= 0")
|
||||
}
|
||||
|
||||
cfg := clone.DefaultConfig()
|
||||
cfg.OutDir = f.out
|
||||
@@ -137,9 +144,11 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
|
||||
cfg.ScopePrefix = f.scopePrefix
|
||||
cfg.ExcludePaths = f.exclude
|
||||
cfg.RespectRobots = !f.noRobots
|
||||
cfg.CrawlDelay = f.crawlDelay
|
||||
cfg.FollowSitemap = !f.noSitemap
|
||||
cfg.Headless = !f.headful
|
||||
cfg.KeepNoscript = f.keepNoscript
|
||||
cfg.MobileReadable = f.mobileReadable
|
||||
cfg.ChromeBin = f.chromeBin
|
||||
cfg.ControlURL = f.controlURL
|
||||
cfg.Resume = !f.noResume
|
||||
|
||||
+36
-5
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/tamnd/kage/sanitize"
|
||||
"github.com/tamnd/kage/urlx"
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
// Logf is an optional sink for human-readable progress lines.
|
||||
@@ -43,9 +44,12 @@ type Cloner struct {
|
||||
mu sync.Mutex
|
||||
seenAssets map[string]bool
|
||||
enqueued int // pages offered to the queue
|
||||
wg sync.WaitGroup
|
||||
pageJobs chan pageItem
|
||||
assetJobs chan assetItem
|
||||
|
||||
crawlLimiter *rate.Limiter
|
||||
|
||||
wg sync.WaitGroup
|
||||
pageJobs chan pageItem
|
||||
assetJobs chan assetItem
|
||||
|
||||
muContent sync.Mutex
|
||||
seenContent map[string]string // sha-256 of page bytes -> first path written
|
||||
@@ -144,6 +148,7 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) {
|
||||
defer func() { _ = c.pool.Close() }()
|
||||
|
||||
c.loadRobots(ctx)
|
||||
c.setupCrawlDelayLimiter()
|
||||
|
||||
// Start workers.
|
||||
var workers sync.WaitGroup
|
||||
@@ -218,6 +223,19 @@ func (c *Cloner) loadRobots(ctx context.Context) {
|
||||
c.robots = robots.Parse(string(data), "kage")
|
||||
}
|
||||
|
||||
func (c *Cloner) setupCrawlDelayLimiter() {
|
||||
delay := c.cfg.CrawlDelay
|
||||
if delay <= 0 && c.cfg.RespectRobots && c.robots != nil {
|
||||
delay = c.robots.CrawlDelay
|
||||
}
|
||||
if delay <= 0 {
|
||||
c.crawlLimiter = nil
|
||||
return
|
||||
}
|
||||
|
||||
c.crawlLimiter = rate.NewLimiter(rate.Every(delay), 1)
|
||||
}
|
||||
|
||||
// seedSitemaps adds in-scope sitemap URLs (from robots and the default path) to
|
||||
// the frontier.
|
||||
func (c *Cloner) seedSitemaps(ctx context.Context) {
|
||||
@@ -252,6 +270,9 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
|
||||
c.stats.skipped.Add(1)
|
||||
return
|
||||
}
|
||||
if !c.waitForCrawlDelay(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
res, err := c.pool.Render(ctx, j.u.String())
|
||||
if err != nil {
|
||||
@@ -305,8 +326,9 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
|
||||
|
||||
asset.RewriteHTML(root, j.u, sink)
|
||||
sanitize.CleanTree(root, sanitize.Options{
|
||||
KeepNoscript: c.cfg.KeepNoscript,
|
||||
Banner: "cloned by kage from " + j.u.String(),
|
||||
KeepNoscript: c.cfg.KeepNoscript,
|
||||
MobileReadable: c.cfg.MobileReadable,
|
||||
Banner: "cloned by kage from " + j.u.String(),
|
||||
})
|
||||
|
||||
var buf strings.Builder
|
||||
@@ -323,6 +345,15 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
|
||||
c.stats.recordPage(c.pagePathKey(j.u), deduped)
|
||||
}
|
||||
|
||||
// waitForCrawlDelay spaces page render starts.
|
||||
func (c *Cloner) waitForCrawlDelay(ctx context.Context) bool {
|
||||
if c.crawlLimiter == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
return c.crawlLimiter.Wait(ctx) == nil
|
||||
}
|
||||
|
||||
// processAsset downloads one asset, rewriting CSS references on the way, and
|
||||
// writes it to its deterministic local path.
|
||||
func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/tamnd/kage/browser"
|
||||
"github.com/tamnd/kage/robots"
|
||||
"github.com/tamnd/kage/urlx"
|
||||
)
|
||||
|
||||
@@ -178,6 +179,56 @@ func TestPageKeyCollapsesDuplicates(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlDelaySpacesPageStarts(t *testing.T) {
|
||||
seed, _ := urlx.ParseSeed("https://ex.com")
|
||||
cfg := DefaultConfig()
|
||||
cfg.RespectRobots = true
|
||||
c := New(seed, cfg, nil)
|
||||
c.robots = &robots.Matcher{CrawlDelay: 20 * time.Millisecond}
|
||||
c.setupCrawlDelayLimiter()
|
||||
|
||||
ctx := context.Background()
|
||||
if !c.waitForCrawlDelay(ctx) {
|
||||
t.Fatal("first crawl-delay wait returned false")
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
if !c.waitForCrawlDelay(ctx) {
|
||||
t.Fatal("second crawl-delay wait returned false")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < 15*time.Millisecond {
|
||||
t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlDelayFlagOverridesRobots(t *testing.T) {
|
||||
seed, _ := urlx.ParseSeed("https://ex.com")
|
||||
cfg := DefaultConfig()
|
||||
cfg.RespectRobots = true
|
||||
cfg.CrawlDelay = 20 * time.Millisecond
|
||||
c := New(seed, cfg, nil)
|
||||
c.robots = &robots.Matcher{CrawlDelay: time.Minute}
|
||||
c.setupCrawlDelayLimiter()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
||||
defer cancel()
|
||||
if !c.waitForCrawlDelay(ctx) {
|
||||
t.Fatal("first crawl-delay wait returned false")
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
if !c.waitForCrawlDelay(ctx) {
|
||||
t.Fatal("second crawl-delay wait returned false")
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
if elapsed < 15*time.Millisecond {
|
||||
t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed)
|
||||
}
|
||||
if elapsed > 150*time.Millisecond {
|
||||
t.Fatalf("second crawl-delay wait = %v, override likely ignored", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func mustURL(t *testing.T, raw string) *url.URL {
|
||||
t.Helper()
|
||||
u, err := url.Parse(raw)
|
||||
|
||||
+8
-6
@@ -56,12 +56,14 @@ type Config struct {
|
||||
ScopePrefix string
|
||||
ExcludePaths []string
|
||||
|
||||
RespectRobots bool
|
||||
FollowSitemap bool
|
||||
Headless bool
|
||||
KeepNoscript bool
|
||||
ChromeBin string
|
||||
ControlURL string
|
||||
RespectRobots bool
|
||||
CrawlDelay time.Duration // override robots.txt Crawl-delay when > 0
|
||||
FollowSitemap bool
|
||||
Headless bool
|
||||
KeepNoscript bool
|
||||
MobileReadable bool
|
||||
ChromeBin string
|
||||
ControlURL string
|
||||
|
||||
// Resume loads the prior run's visited set and skips pages already written,
|
||||
// so an interrupted or repeated clone picks up where it left off instead of
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: "Installation"
|
||||
description: "Install kage from Go, Homebrew, a release archive, a Linux package, or the container image, and point it at a browser."
|
||||
description: "Install kage from Go, Homebrew, Scoop, a release archive, a Linux package, or the container image, and point it at a browser."
|
||||
weight: 20
|
||||
---
|
||||
|
||||
@@ -12,10 +12,38 @@ kage is a single binary. Pick whichever channel suits you.
|
||||
go install github.com/tamnd/kage/cmd/kage@latest
|
||||
```
|
||||
|
||||
## Homebrew
|
||||
## Homebrew (macOS)
|
||||
|
||||
```bash
|
||||
brew install tamnd/tap/kage
|
||||
brew install --cask tamnd/tap/kage
|
||||
```
|
||||
|
||||
The cask installs the prebuilt macOS binary. On Linux, use the packages below or
|
||||
`go install`.
|
||||
|
||||
## Scoop (Windows)
|
||||
|
||||
```bash
|
||||
scoop bucket add tamnd https://github.com/tamnd/scoop-bucket
|
||||
scoop install kage
|
||||
```
|
||||
|
||||
## Linux (apt and dnf)
|
||||
|
||||
A signed apt and dnf repository tracks every release, so `apt upgrade` and
|
||||
`dnf upgrade` keep kage current.
|
||||
|
||||
```bash
|
||||
# Debian, Ubuntu
|
||||
curl -fsSL https://tamnd.github.io/linux-repo/gpg.key \
|
||||
| sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" \
|
||||
| sudo tee /etc/apt/sources.list.d/tamnd.list
|
||||
sudo apt update && sudo apt install kage
|
||||
|
||||
# Fedora, RHEL
|
||||
sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo
|
||||
sudo dnf install kage
|
||||
```
|
||||
|
||||
## Release archives and Linux packages
|
||||
@@ -24,14 +52,14 @@ Every [release](https://github.com/tamnd/kage/releases) attaches `tar.gz`
|
||||
archives (and a `.zip` for Windows) for Linux, macOS, Windows, and FreeBSD, plus
|
||||
`.deb`, `.rpm`, and `.apk` packages and a `checksums.txt` with a cosign
|
||||
signature. Download the one for your platform, extract `kage`, and put it on your
|
||||
`PATH`.
|
||||
`PATH`. To install a package directly without the repo above:
|
||||
|
||||
```bash
|
||||
# Debian/Ubuntu
|
||||
sudo dpkg -i kage_*_linux_amd64.deb
|
||||
sudo dpkg -i kage_*_amd64.deb
|
||||
|
||||
# Fedora/RHEL
|
||||
sudo rpm -i kage_*_linux_amd64.rpm
|
||||
sudo rpm -i kage-*.x86_64.rpm
|
||||
```
|
||||
|
||||
## Container
|
||||
|
||||
@@ -41,7 +41,18 @@ kage open paulgraham.com.zim # read it back with kage
|
||||
kiwix-serve paulgraham.com.zim # or serve it with Kiwix at http://localhost
|
||||
```
|
||||
|
||||
You can also double-click the file in the [Kiwix desktop app](https://kiwix.org/en/applications/), or load it on Kiwix for Android or iOS to read your mirror on your phone. kage writes the metadata the format and `zimcheck` treat as mandatory, including the title, description, and the favicon Kiwix shows as the book icon in its library, so the archive shows up properly rather than as an untitled, iconless entry. One caveat: kage does not write the full-text search index that Kiwix's own packs ship with, so browsing works everywhere while in-reader search is limited.
|
||||
You can also double-click the file in the [Kiwix desktop app](https://kiwix.org/en/applications/), or load it on Kiwix for Android or iOS to read your mirror on your phone. kage writes the metadata the format and `zimcheck` treat as mandatory, including the title, description, and the favicon Kiwix shows as the book icon in its library, so the archive shows up properly rather than as an untitled, iconless entry.
|
||||
|
||||
### Searching a packed mirror
|
||||
|
||||
Each page is stored under its own real `<title>`, so the search box in Kiwix (and any other ZIM reader) suggests pages by their readable title: type `five` into a paulgraham.com archive and it offers "Five Founders" and "Female Founders", not a filename. This title search works in every reader with no extra index.
|
||||
|
||||
What kage does not write is a Xapian full-text index, the separate search database that lets Kiwix match words inside a page's body. Xapian is GPL and kage is MIT, so linking it in would change kage's license; rather than do that, kage leaves full-text search to its own columnar export. `kage parquet export mirror.zim` writes one row per page with a `text` column, which [DuckDB](https://duckdb.org) searches with its `fts` extension or a plain `ILIKE`, fully offline and ranked:
|
||||
|
||||
```bash
|
||||
kage parquet export paulgraham.com.zim -o paulgraham.parquet
|
||||
duckdb -c "SELECT url FROM 'paulgraham.parquet' WHERE text ILIKE '%schlep blindness%'"
|
||||
```
|
||||
|
||||
## A self-contained binary
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ images, and fonts, and writes a browsable mirror to `<out>/<host>/`.
|
||||
| Flag | Default | Meaning |
|
||||
|------|---------|---------|
|
||||
| `--no-robots` | `false` | Ignore `robots.txt` |
|
||||
| `--crawl-delay` | `0s` | Override robots.txt `Crawl-delay` between page starts (0 = use robots.txt) |
|
||||
| `--no-sitemap` | `false` | Do not seed URLs from `sitemap.xml` |
|
||||
| `--user-agent` | Chrome UA | User-Agent for asset and robots fetches |
|
||||
|
||||
|
||||
@@ -6,6 +6,12 @@ weight: 40
|
||||
|
||||
The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version.
|
||||
|
||||
## v0.3.9
|
||||
|
||||
A fix for the antivirus warning some Windows users hit when installing kage.
|
||||
|
||||
- **The Windows build no longer ships the leakless helper antivirus flags.** kage renders pages with [go-rod](https://github.com/go-rod/rod), whose launcher pulls in [leakless](https://github.com/ysmood/leakless), a small watchdog that force-kills Chrome if kage exits. leakless carries a prebuilt helper binary for every platform and links the Windows one straight into `kage.exe`. Windows Defender recognises that helper as `Trojan:Win32/Kepavll!rfn` and quarantines it, so a fresh `scoop install` failed with a virus warning on `leakless.exe` ([#68](https://github.com/tamnd/kage/issues/68)). kage already launches Chrome with leakless switched off, so the helper never ran anyway. It is now replaced with a stub that carries no embedded binary, which drops about 1.28 MB from the Windows build and clears the warning. Thanks to John Pywtorak for the report. `go install`, unaffected before, stays clean.
|
||||
|
||||
## v0.3.4
|
||||
|
||||
Two community fixes: a clean stop for `kage serve`, and pages with heavy JavaScript that used to be dropped.
|
||||
|
||||
@@ -14,6 +14,7 @@ require (
|
||||
github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6
|
||||
golang.org/x/image v0.42.0
|
||||
golang.org/x/net v0.56.0
|
||||
golang.org/x/time v0.15.0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -53,3 +54,11 @@ require (
|
||||
golang.org/x/text v0.38.0 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
)
|
||||
|
||||
// go-rod's launcher imports github.com/ysmood/leakless, which base64/gzip-embeds
|
||||
// a prebuilt leakless.exe into the Windows build. Antivirus engines flag that
|
||||
// embedded helper as malware and quarantine kage on install (issue #68). kage
|
||||
// always launches Chrome with leakless disabled (browser/leakless.go), so the
|
||||
// guard is dead weight; this replace swaps in an API-compatible stub that
|
||||
// carries no embedded binary.
|
||||
replace github.com/ysmood/leakless => ./third_party/leakless
|
||||
|
||||
@@ -107,9 +107,6 @@ github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
|
||||
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
|
||||
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
|
||||
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
|
||||
github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
||||
github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
|
||||
github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
@@ -123,6 +120,8 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
|
||||
@@ -354,3 +354,80 @@ func TestHandler(t *testing.T) {
|
||||
t.Errorf("GET missing = %d, want 404", code)
|
||||
}
|
||||
}
|
||||
|
||||
// titleOf returns the stored entry title for a content url, scanning entries in
|
||||
// url order since the reader exposes titles only through EntryAt.
|
||||
func titleOf(t *testing.T, r *zim.Reader, url string) string {
|
||||
t.Helper()
|
||||
for i := uint32(0); i < r.Count(); i++ {
|
||||
e, err := r.EntryAt(i)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if e.Namespace == zim.NamespaceContent && e.URL == url {
|
||||
return e.Title
|
||||
}
|
||||
}
|
||||
t.Fatalf("no content entry for %q", url)
|
||||
return ""
|
||||
}
|
||||
|
||||
// TestBuildZIMPageTitles checks that each HTML page entry carries its own
|
||||
// <title> (collapsed to one line), that a page with no title falls back to its
|
||||
// url, and that a non-HTML asset keeps the url as its title.
|
||||
func TestBuildZIMPageTitles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
host := filepath.Join(root, "example.com")
|
||||
files := map[string]string{
|
||||
"index.html": "<!doctype html><title>Home</title><h1>Hi</h1>",
|
||||
"essay/index.html": "<!doctype html><title>\n A Long\n Title </title><p>x</p>",
|
||||
"bare/index.html": "<!doctype html><h1>No title here</h1>",
|
||||
"logo.png": "\x89PNGfake",
|
||||
}
|
||||
for rel, body := range files {
|
||||
p := filepath.Join(host, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
out := filepath.Join(t.TempDir(), "titles.zim")
|
||||
if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14"}); err != nil {
|
||||
t.Fatalf("BuildZIM: %v", err)
|
||||
}
|
||||
r, err := zim.Open(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer func() { _ = r.Close() }()
|
||||
|
||||
cases := map[string]string{
|
||||
"index.html": "Home",
|
||||
"essay/index.html": "A Long Title", // newlines and runs collapsed to single spaces
|
||||
"bare/index.html": "bare/index.html",
|
||||
"logo.png": "logo.png",
|
||||
}
|
||||
for url, want := range cases {
|
||||
if got := titleOf(t, r, url); got != want {
|
||||
t.Errorf("title of %q = %q, want %q", url, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollapseSpaces(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
" hello world ": "hello world",
|
||||
"line\n\tone\r\n two": "line one two",
|
||||
"": "",
|
||||
" ": "",
|
||||
"single": "single",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := collapseSpaces(in); got != want {
|
||||
t.Errorf("collapseSpaces(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+24
-5
@@ -170,11 +170,16 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, *clusterCache,
|
||||
return err
|
||||
}
|
||||
mime := MimeForExt(rel)
|
||||
title := ""
|
||||
if mime == "text/html" {
|
||||
htmlPages = append(htmlPages, rel)
|
||||
// Store the page's real <title> on its entry, not the URL path, so a
|
||||
// ZIM reader's search box and suggestions match the readable title.
|
||||
// An empty result leaves the writer to fall back to the url.
|
||||
title = htmlTitleOfBytes(data)
|
||||
}
|
||||
counts[mime]++
|
||||
w.AddContent(zim.NamespaceContent, rel, "", mime, data)
|
||||
w.AddContent(zim.NamespaceContent, rel, title, mime, data)
|
||||
return nil
|
||||
})
|
||||
if walkErr != nil {
|
||||
@@ -240,16 +245,30 @@ func htmlTitleOf(mirrorDir, mainURL string) string {
|
||||
if mainURL == "" {
|
||||
return ""
|
||||
}
|
||||
f, err := os.Open(filepath.Join(mirrorDir, filepath.FromSlash(mainURL)))
|
||||
data, err := os.ReadFile(filepath.Join(mirrorDir, filepath.FromSlash(mainURL)))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
doc, err := html.Parse(f)
|
||||
return htmlTitleOfBytes(data)
|
||||
}
|
||||
|
||||
// htmlTitleOfBytes returns the first <title> in the given HTML with surrounding
|
||||
// and internal whitespace collapsed, or "" if the bytes do not parse or carry
|
||||
// no title. Page entries use it so a ZIM reader shows the page's real title
|
||||
// instead of its URL path.
|
||||
func htmlTitleOfBytes(data []byte) string {
|
||||
doc, err := html.Parse(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(findTitle(doc))
|
||||
return collapseSpaces(findTitle(doc))
|
||||
}
|
||||
|
||||
// collapseSpaces trims s and replaces every run of whitespace (including the
|
||||
// newlines a wrapped <title> can carry) with a single space, so a title reads
|
||||
// as one clean line in a reader's library and search results.
|
||||
func collapseSpaces(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
|
||||
// findTitle returns the text of the first <title> element in depth-first order.
|
||||
|
||||
@@ -29,6 +29,14 @@ type Options struct {
|
||||
// Banner, when non-empty, is inserted as an HTML comment at the top of the
|
||||
// document.
|
||||
Banner string
|
||||
// MobileReadable injects a viewport meta tag and a small CSS block that
|
||||
// makes legacy, font-era sites readable on mobile. It is intended for
|
||||
// archives of 1990s/2000s sites that use <font size="2">, table layouts,
|
||||
// and no viewport declaration — all of which render as microscopic text on
|
||||
// a phone. The injected CSS overrides font sizes, loosens line height, caps
|
||||
// the content width, and hides image-map navigation elements that are
|
||||
// useless offline.
|
||||
MobileReadable bool
|
||||
}
|
||||
|
||||
// Report counts what was removed, for the run summary and for tests.
|
||||
@@ -72,6 +80,10 @@ func CleanTree(root *html.Node, opts Options) Report {
|
||||
var rep Report
|
||||
clean(root, opts, &rep)
|
||||
rep.CharsetAdded = ensureCharset(root)
|
||||
if opts.MobileReadable {
|
||||
ensureViewport(root)
|
||||
injectMobileCSS(root)
|
||||
}
|
||||
if opts.Banner != "" {
|
||||
insertBanner(root, opts.Banner)
|
||||
}
|
||||
@@ -291,6 +303,83 @@ func findElement(n *html.Node, a atom.Atom) *html.Node {
|
||||
return nil
|
||||
}
|
||||
|
||||
// mobileCSS is injected when MobileReadable is set. It rewrites font-era
|
||||
// HTML for comfortable reading on a phone. Key rules:
|
||||
//
|
||||
// - box-sizing:border-box — makes padding predictable in a layout built with
|
||||
// HTML width attributes, so our padding doesn't cause overflow.
|
||||
// - body — no fixed max-width here; let the table rules handle width instead.
|
||||
// overflow-x:hidden catches any stray overflow without a scrollbar.
|
||||
// - font element — overrides in-HTML size/face attributes (e.g. <font size="2">).
|
||||
// - [width],[height] — cancels all HTML attribute widths/heights on every
|
||||
// element (tables, tds, imgs, etc.) so fixed-pixel columns become fluid.
|
||||
// - table — fluid, auto layout, no horizontal scroll.
|
||||
// - td — auto width so a three-column table (nav | spacer | content) collapses
|
||||
// to one usable column once the nav td is hidden.
|
||||
// - img — responsive: never wider than its container.
|
||||
// - td:has(>img[usemap]) — hides the entire nav column td, not just the image
|
||||
// inside it; hiding only the img left a tall empty white box.
|
||||
// - td:has(>img[src*="trans_1x1"]) — hides the 26 px spacer column td (a 1×1
|
||||
// transparent GIF whose only job was spacing in the original table layout).
|
||||
const mobileCSS = `*{box-sizing:border-box}` +
|
||||
`:root{font-size:18px}` +
|
||||
`body{margin:0;padding:.75em 1em;line-height:1.7;font-family:Georgia,"Times New Roman",serif;overflow-x:hidden}` +
|
||||
`font{font-size:1rem!important;font-family:inherit!important;color:inherit!important}` +
|
||||
`[width]{width:auto!important;max-width:100%!important}` +
|
||||
`[height]{height:auto!important}` +
|
||||
`table{width:100%!important;max-width:100%!important;table-layout:auto!important;border-collapse:collapse!important;word-break:break-word}` +
|
||||
`td,th{width:auto!important;max-width:100%!important;padding:.35em .5em!important;vertical-align:top!important;overflow-wrap:break-word}` +
|
||||
`img{max-width:100%!important;height:auto!important}` +
|
||||
`img[usemap],map{display:none!important}` +
|
||||
`td:has(>img[usemap]),td:has(>map){display:none!important}` +
|
||||
`img[src*="trans_1x1"],img[src*="spacer"],img[height="1"],img[width="1"]{display:none!important}` +
|
||||
`td:has(>img[src*="trans_1x1"]:only-child),td:has(>img[height="1"]:only-child){display:none!important}`
|
||||
|
||||
// ensureViewport inserts <meta name="viewport" content="width=device-width,
|
||||
// initial-scale=1"> at the top of <head> when the document does not already
|
||||
// carry one. Without it a mobile browser shrinks the page to fit the screen
|
||||
// at desktop scale, making text unreadably small regardless of CSS font sizes.
|
||||
func ensureViewport(root *html.Node) {
|
||||
head := findElement(root, atom.Head)
|
||||
if head == nil {
|
||||
return
|
||||
}
|
||||
// Check whether a viewport meta already exists.
|
||||
for c := head.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type == html.ElementNode && c.DataAtom == atom.Meta &&
|
||||
strings.EqualFold(attr(c, "name"), "viewport") {
|
||||
return
|
||||
}
|
||||
}
|
||||
meta := &html.Node{
|
||||
Type: html.ElementNode,
|
||||
Data: "meta",
|
||||
DataAtom: atom.Meta,
|
||||
Attr: []html.Attribute{
|
||||
{Key: "name", Val: "viewport"},
|
||||
{Key: "content", Val: "width=device-width, initial-scale=1"},
|
||||
},
|
||||
}
|
||||
head.InsertBefore(meta, head.FirstChild)
|
||||
}
|
||||
|
||||
// injectMobileCSS appends a <style> block containing mobileCSS to <head>.
|
||||
// It goes at the end of <head> so it wins specificity ties over any existing
|
||||
// inline styles the page already carries.
|
||||
func injectMobileCSS(root *html.Node) {
|
||||
head := findElement(root, atom.Head)
|
||||
if head == nil {
|
||||
return
|
||||
}
|
||||
style := &html.Node{
|
||||
Type: html.ElementNode,
|
||||
Data: "style",
|
||||
DataAtom: atom.Style,
|
||||
}
|
||||
style.AppendChild(&html.Node{Type: html.TextNode, Data: mobileCSS})
|
||||
head.AppendChild(style)
|
||||
}
|
||||
|
||||
// insertBanner prepends an HTML comment to the document.
|
||||
func insertBanner(root *html.Node, text string) {
|
||||
c := &html.Node{Type: html.CommentNode, Data: " " + text + " "}
|
||||
|
||||
@@ -197,6 +197,74 @@ func TestCharsetAddedWhenMissing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMobileReadableInjectsViewportAndCSS(t *testing.T) {
|
||||
// A paulgraham.com-style page: no viewport, tiny <font size="2"> markup.
|
||||
in := `<html><head><title>Essay</title></head>` +
|
||||
`<body><font size="2" face="verdana"><p>Hello world.</p></font></body></html>`
|
||||
out, _, err := Strip([]byte(in), Options{MobileReadable: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(out)
|
||||
if !strings.Contains(s, `name="viewport"`) {
|
||||
t.Error("viewport meta not injected")
|
||||
}
|
||||
if !strings.Contains(s, "width=device-width") {
|
||||
t.Error("viewport content wrong")
|
||||
}
|
||||
if !strings.Contains(s, "font-size:18px") {
|
||||
t.Error("mobile CSS not injected")
|
||||
}
|
||||
if !strings.Contains(s, "font{font-size:1rem") {
|
||||
t.Error("font override not in mobile CSS")
|
||||
}
|
||||
// Fluid table rules must be present.
|
||||
if !strings.Contains(s, "table{width:100%") {
|
||||
t.Error("fluid table rule missing")
|
||||
}
|
||||
// [width] override must be present so fixed HTML width attributes are cancelled.
|
||||
if !strings.Contains(s, "[width]{width:auto") {
|
||||
t.Error("[width] override missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMobileReadableHidesNavColumnTd(t *testing.T) {
|
||||
// paulgraham.com wraps its image-map nav in a <td>. Hiding only the img
|
||||
// leaves a tall empty box; the CSS must also target the containing td.
|
||||
in := `<html><head><title>T</title></head><body>` +
|
||||
`<table><tr>` +
|
||||
`<td><map name="nav"><area shape="rect" coords="0,0,67,21" href="index.html"></map>` +
|
||||
`<img src="nav.gif" width="69" height="357" usemap="#nav"/></td>` +
|
||||
`<td><img src="https://cdn.example.com/trans_1x1.gif" height="1" width="26"/></td>` +
|
||||
`<td><p>Essay text.</p></td>` +
|
||||
`</tr></table>` +
|
||||
`</body></html>`
|
||||
out, _, err := Strip([]byte(in), Options{MobileReadable: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(out)
|
||||
// td:has(>img[usemap]) rule must be present so the whole nav column is hidden.
|
||||
if !strings.Contains(s, "td:has(>img[usemap])") {
|
||||
t.Error("td:has(>img[usemap]) rule missing from mobile CSS")
|
||||
}
|
||||
// td:has(>img[src*="trans_1x1"]) rule must be present for spacer column.
|
||||
if !strings.Contains(s, `td:has(>img[src*="trans_1x1"]`) {
|
||||
t.Error(`td:has(>img[src*="trans_1x1"]) spacer-column rule missing from mobile CSS`)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMobileReadableSkipsExistingViewport(t *testing.T) {
|
||||
in := `<html><head><meta name="viewport" content="width=device-width"><title>x</title></head><body></body></html>`
|
||||
out, _, err := Strip([]byte(in), Options{MobileReadable: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := strings.Count(string(out), `name="viewport"`); n != 1 {
|
||||
t.Errorf("viewport injected when one already existed (count %d)", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCharsetNotDuplicated(t *testing.T) {
|
||||
// A page that already declares a charset, in either form, is left alone.
|
||||
cases := []string{
|
||||
|
||||
Vendored
+3
@@ -0,0 +1,3 @@
|
||||
module github.com/ysmood/leakless
|
||||
|
||||
go 1.26.4
|
||||
Vendored
+55
@@ -0,0 +1,55 @@
|
||||
// Package leakless is kage's drop-in replacement for
|
||||
// github.com/ysmood/leakless, wired in through a replace directive in the root
|
||||
// go.mod.
|
||||
//
|
||||
// The upstream package guards a child process by extracting a small helper
|
||||
// executable that force-kills the child when the parent dies. It ships that
|
||||
// helper by base64/gzip-embedding a prebuilt binary for every target
|
||||
// (bin_amd64_windows.go and friends), so the packed leakless.exe ends up linked
|
||||
// into any program that imports the package, kage included. Antivirus engines
|
||||
// flag that embedded Windows helper as malware, so a fresh install of kage got
|
||||
// quarantined before it ever ran (issue #68).
|
||||
//
|
||||
// kage already launches Chrome with leakless disabled (see
|
||||
// browser/leakless.go), so the guard is never used. This stub keeps the exact
|
||||
// public surface go-rod's launcher depends on (New, Support, LockPort, and the
|
||||
// Launcher type's Command/Pid/Err) while carrying no embedded binary, which
|
||||
// removes the false positive entirely. Support reports no guard is available,
|
||||
// so go-rod's launcher never takes the leakless path even if a caller asked
|
||||
// for it.
|
||||
package leakless
|
||||
|
||||
import "os/exec"
|
||||
|
||||
// Launcher mirrors the upstream type. The channel is left unbuffered and is
|
||||
// never written to, matching the "may never receive the pid" contract go-rod
|
||||
// already tolerates.
|
||||
type Launcher struct {
|
||||
pid chan int
|
||||
}
|
||||
|
||||
// New returns a Launcher. It allocates nothing beyond the pid channel.
|
||||
func New() *Launcher {
|
||||
return &Launcher{pid: make(chan int)}
|
||||
}
|
||||
|
||||
// Command builds the command without a guard wrapper. Because Support returns
|
||||
// false, go-rod never calls this in practice; if some other caller did, running
|
||||
// the target directly is the correct no-guard behaviour.
|
||||
func (l *Launcher) Command(name string, arg ...string) *exec.Cmd {
|
||||
return exec.Command(name, arg...)
|
||||
}
|
||||
|
||||
// Pid returns the (never-signalled) pid channel.
|
||||
func (l *Launcher) Pid() chan int { return l.pid }
|
||||
|
||||
// Err returns the guard error, always empty here since there is no guard.
|
||||
func (l *Launcher) Err() string { return "" }
|
||||
|
||||
// Support reports whether a guard binary is available. It always returns false
|
||||
// so callers skip leakless entirely.
|
||||
func Support() bool { return false }
|
||||
|
||||
// LockPort is the cross-process mutex the upstream guard uses to serialise
|
||||
// extraction. With no guard there is nothing to serialise, so it is a no-op.
|
||||
func LockPort(port int) func() { return func() {} }
|
||||
Reference in New Issue
Block a user