From 9c9576b5b9b40269ac436eb7877267488d28c934 Mon Sep 17 00:00:00 2001 From: wehub-resource-sync Date: Tue, 14 Jul 2026 10:24:05 +0800 Subject: [PATCH] chore: import upstream snapshot with attribution --- .github/workflows/ci.yml | 146 +++++ .github/workflows/docs.yml | 135 +++++ .github/workflows/release.yml | 112 ++++ .gitignore | 11 + .gitmodules | 3 + .goreleaser.yaml | 247 ++++++++ CHANGELOG.md | 278 +++++++++ Dockerfile | 47 ++ LICENSE | 21 + Makefile | 45 ++ README.md | 284 +++++++++ README.wehub.md | 7 + asset/asset_test.go | 118 ++++ asset/css.go | 69 +++ asset/download.go | 185 ++++++ asset/download_test.go | 175 ++++++ asset/html.go | 161 ++++++ browser/leakless.go | 7 + browser/pool.go | 489 ++++++++++++++++ browser/pool_test.go | 210 +++++++ cli/clone.go | 260 +++++++++ cli/open.go | 66 +++ cli/pack.go | 435 ++++++++++++++ cli/parquet.go | 94 +++ cli/root.go | 109 ++++ cli/serve.go | 75 +++ cli/styles.go | 14 + cli/version.go | 9 + clone/cloner.go | 570 +++++++++++++++++++ clone/cloner_test.go | 425 ++++++++++++++ clone/config.go | 146 +++++ clone/dedup_test.go | 69 +++ clone/frontier.go | 106 ++++ clone/frontier_test.go | 67 +++ clone/sitemap.go | 83 +++ clone/stats.go | 111 ++++ clone/wantasset_test.go | 70 +++ cmd/kage/main.go | 31 + dataset/dataset.go | 307 ++++++++++ dataset/dataset_test.go | 115 ++++ docs/content/_index.md | 38 ++ docs/content/getting-started/_index.md | 11 + docs/content/getting-started/installation.md | 91 +++ docs/content/getting-started/introduction.md | 47 ++ docs/content/getting-started/quick-start.md | 68 +++ docs/content/guides/_index.md | 11 + docs/content/guides/packing-a-mirror.md | 165 ++++++ docs/content/guides/resuming-a-run.md | 48 ++ docs/content/guides/scoping-a-crawl.md | 73 +++ docs/content/guides/serving-a-mirror.md | 48 ++ docs/content/reference/_index.md | 11 + docs/content/reference/cli.md | 129 +++++ docs/content/reference/configuration.md | 69 +++ docs/content/reference/release-notes.md | 93 +++ docs/demo/kage.tape | 34 ++ docs/static/demo.gif | Bin 0 -> 353624 bytes docs/static/favicon.svg | 1 + docs/static/webview.png | Bin 0 -> 645343 bytes docs/tago.toml | 10 + go.mod | 64 +++ go.sum | 131 +++++ pack/appbundle.go | 161 ++++++ pack/appbundle_test.go | 131 +++++ pack/appdir.go | 156 +++++ pack/appdir_test.go | 125 ++++ pack/binary.go | 74 +++ pack/cache.go | 134 +++++ pack/cache_test.go | 122 ++++ pack/embed.go | 50 ++ pack/icns.go | 92 +++ pack/icns_test.go | 73 +++ pack/icon.go | 329 +++++++++++ pack/icon_test.go | 212 +++++++ pack/mime.go | 52 ++ pack/osdetect.go | 57 ++ pack/osdetect_test.go | 40 ++ pack/pack_test.go | 433 ++++++++++++++ pack/serve.go | 55 ++ pack/zim.go | 325 +++++++++++ robots/robots.go | 218 +++++++ robots/robots_test.go | 73 +++ sanitize/sanitize.go | 401 +++++++++++++ sanitize/sanitize_test.go | 286 ++++++++++ scripts/ensure_cloudflare_pages.py | 144 +++++ third_party/leakless/go.mod | 3 + third_party/leakless/leakless.go | 55 ++ urlx/urlx.go | 413 ++++++++++++++ urlx/urlx_test.go | 292 ++++++++++ viewer/browser.go | 41 ++ viewer/browser_test.go | 37 ++ viewer/viewer.go | 24 + viewer/webview.go | 53 ++ zim/codec.go | 54 ++ zim/format.go | 115 ++++ zim/reader.go | 400 +++++++++++++ zim/writer.go | 421 ++++++++++++++ zim/zim_test.go | 136 +++++ 97 files changed, 12841 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 .goreleaser.yaml create mode 100644 CHANGELOG.md create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 README.wehub.md create mode 100644 asset/asset_test.go create mode 100644 asset/css.go create mode 100644 asset/download.go create mode 100644 asset/download_test.go create mode 100644 asset/html.go create mode 100644 browser/leakless.go create mode 100644 browser/pool.go create mode 100644 browser/pool_test.go create mode 100644 cli/clone.go create mode 100644 cli/open.go create mode 100644 cli/pack.go create mode 100644 cli/parquet.go create mode 100644 cli/root.go create mode 100644 cli/serve.go create mode 100644 cli/styles.go create mode 100644 cli/version.go create mode 100644 clone/cloner.go create mode 100644 clone/cloner_test.go create mode 100644 clone/config.go create mode 100644 clone/dedup_test.go create mode 100644 clone/frontier.go create mode 100644 clone/frontier_test.go create mode 100644 clone/sitemap.go create mode 100644 clone/stats.go create mode 100644 clone/wantasset_test.go create mode 100644 cmd/kage/main.go create mode 100644 dataset/dataset.go create mode 100644 dataset/dataset_test.go create mode 100644 docs/content/_index.md create mode 100644 docs/content/getting-started/_index.md create mode 100644 docs/content/getting-started/installation.md create mode 100644 docs/content/getting-started/introduction.md create mode 100644 docs/content/getting-started/quick-start.md create mode 100644 docs/content/guides/_index.md create mode 100644 docs/content/guides/packing-a-mirror.md create mode 100644 docs/content/guides/resuming-a-run.md create mode 100644 docs/content/guides/scoping-a-crawl.md create mode 100644 docs/content/guides/serving-a-mirror.md create mode 100644 docs/content/reference/_index.md create mode 100644 docs/content/reference/cli.md create mode 100644 docs/content/reference/configuration.md create mode 100644 docs/content/reference/release-notes.md create mode 100644 docs/demo/kage.tape create mode 100644 docs/static/demo.gif create mode 100644 docs/static/favicon.svg create mode 100644 docs/static/webview.png create mode 100644 docs/tago.toml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pack/appbundle.go create mode 100644 pack/appbundle_test.go create mode 100644 pack/appdir.go create mode 100644 pack/appdir_test.go create mode 100644 pack/binary.go create mode 100644 pack/cache.go create mode 100644 pack/cache_test.go create mode 100644 pack/embed.go create mode 100644 pack/icns.go create mode 100644 pack/icns_test.go create mode 100644 pack/icon.go create mode 100644 pack/icon_test.go create mode 100644 pack/mime.go create mode 100644 pack/osdetect.go create mode 100644 pack/osdetect_test.go create mode 100644 pack/pack_test.go create mode 100644 pack/serve.go create mode 100644 pack/zim.go create mode 100644 robots/robots.go create mode 100644 robots/robots_test.go create mode 100644 sanitize/sanitize.go create mode 100644 sanitize/sanitize_test.go create mode 100755 scripts/ensure_cloudflare_pages.py create mode 100644 third_party/leakless/go.mod create mode 100644 third_party/leakless/leakless.go create mode 100644 urlx/urlx.go create mode 100644 urlx/urlx_test.go create mode 100644 viewer/browser.go create mode 100644 viewer/browser_test.go create mode 100644 viewer/viewer.go create mode 100644 viewer/webview.go create mode 100644 zim/codec.go create mode 100644 zim/format.go create mode 100644 zim/reader.go create mode 100644 zim/writer.go create mode 100644 zim/zim_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e050cb1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,146 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +# Cancel an in-flight run when a branch is pushed again. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Build and test on Linux and macOS with the race detector on. gofmt and vet + # run here too so a formatting slip fails fast on both platforms. The full + # suite includes Chrome-driven end-to-end tests, so the runners install a + # browser first; tests that find none skip themselves, so the job still passes + # if a future runner image drops Chrome. + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7.0.0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - name: install chromium (linux) + if: matrix.os == 'ubuntu-latest' + uses: browser-actions/setup-chrome@v2.1.2 + id: chrome + - name: gofmt + run: | + unformatted=$(gofmt -s -l .) + if [ -n "$unformatted" ]; then + echo "These files need gofmt -s -w:" + echo "$unformatted" + exit 1 + fi + - name: go vet + run: go vet ./... + - name: build + run: go build ./... + # The GitHub Ubuntu runner disables unprivileged user namespaces (AppArmor), + # so Chrome's sandbox cannot initialize there and the secure default would + # make it refuse to start. IN_DOCKER is kage's documented escape hatch for + # exactly that case, and setting it here also exercises the container path. + # macOS does not need it, so it stays empty on that leg. + - name: test + env: + KAGE_CHROME: ${{ steps.chrome.outputs.chrome-path }} + IN_DOCKER: ${{ matrix.os == 'ubuntu-latest' && '1' || '' }} + run: go test -race -count=1 -coverprofile=coverage.out ./... + - name: coverage summary + if: matrix.os == 'ubuntu-latest' + run: go tool cover -func=coverage.out | tail -1 + + # golangci-lint bundles staticcheck, govet, ineffassign, errcheck, unused and + # more, so it is the main quality gate. + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - uses: golangci/golangci-lint-action@v9.2.1 + with: + version: latest + + # Scan the module and its dependencies for known vulnerabilities. + govulncheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - name: govulncheck + run: | + go install golang.org/x/vuln/cmd/govulncheck@latest + govulncheck ./... + + # Confirm go.mod and go.sum are tidy: a PR that adds an import without running + # go mod tidy fails here instead of breaking a later release. + tidy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - name: go mod tidy is clean + run: | + go mod tidy + git diff --exit-code -- go.mod go.sum + + # Compile the optional native-window viewer (-tags webview, cgo) so that path + # keeps building. The default CI build is pure Go and never touches it. The + # viewer code is the same Go on every OS, only the system WebView library + # differs, so a macOS compile (WebKit ships in the SDK) catches our + # regressions without the WebKitGTK version juggling Linux runners need. It is + # build-only: actually opening a window needs a display. + webview: + runs-on: macos-latest + steps: + - uses: actions/checkout@v7.0.0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - name: build webview viewer + run: CGO_ENABLED=1 go build -tags webview ./cmd/kage + + # Cross-compile the GUI-subsystem Windows base the release ships for + # double-click viewers (kage pack --base). A change that breaks the + # -H windowsgui link is caught here instead of at release time. Pure Go, so it + # cross-compiles from Linux with no extra toolchain. + windows-gui: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - name: build windowsgui base + env: + GOOS: windows + CGO_ENABLED: "0" + run: go build -ldflags "-H=windowsgui" -o kage-windowsgui.exe ./cmd/kage diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..4fa818a --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,135 @@ +name: Docs + +on: + push: + branches: [main] + paths: + - "docs/**" + - ".github/workflows/docs.yml" + - "scripts/ensure_cloudflare_pages.py" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + deployments: write + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +concurrency: + group: docs-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + with: + submodules: true + # Sitemap lastmod comes from the latest content commit. + fetch-depth: 0 + + - name: Checkout tago + uses: actions/checkout@v7.0.0 + with: + repository: tamnd/tago + path: .tago-src + fetch-depth: 1 + + - name: Setup Go + uses: actions/setup-go@v6.4.0 + with: + go-version-file: .tago-src/go.mod + cache: false + + - name: Cache tago binary + id: tago-bin-cache + uses: actions/cache@v5.0.5 + with: + path: /usr/local/bin/tago + key: tago-bin-${{ runner.os }}-${{ hashFiles('.tago-src/go.sum', '.tago-src/**/*.go') }} + + - name: Build tago + if: steps.tago-bin-cache.outputs.cache-hit != 'true' + run: | + cd .tago-src + go build -o /usr/local/bin/tago ./cmd/tago/ + + # Two builds, one per deploy target. The theme links every page and asset + # with absURL, so each output carries its own base path: the Pages build + # gets the /kage/ sub-path, the Cloudflare build gets the domain root. + - name: Build for GitHub Pages + working-directory: docs + run: tago build --base-url "https://tamnd.github.io/kage/" --output public-pages + + # tago absURLs theme links but passes markdown content links through, so + # root-relative links in content need the sub-path prefixed for the Pages + # mirror. The Cloudflare build serves from the root and needs none. + - name: Rewrite content links for the Pages sub-path + run: | + find docs/public-pages -name '*.html' -print0 | + xargs -0 perl -pi -e 's|(href=")/(?!/)|${1}/kage/|g; s|(src=")/(?!/)|${1}/kage/|g' + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v5.0.0 + with: + path: docs/public-pages + + - name: Build for Cloudflare Pages + working-directory: docs + run: tago build --base-url "https://kage.tamnd.com/" --output public-cf + + - name: Upload Cloudflare artifact + uses: actions/upload-artifact@v7.0.1 + with: + name: public-cf + path: docs/public-cf + retention-days: 1 + + deploy-pages: + runs-on: ubuntu-latest + needs: build + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v5.0.0 + + deploy-cloudflare: + runs-on: ubuntu-latest + needs: build + concurrency: + group: cloudflare-pages-kage + cancel-in-progress: true + steps: + - uses: actions/checkout@v7.0.0 + with: + fetch-depth: 1 + sparse-checkout: scripts/ + + - name: Download Cloudflare artifact + uses: actions/download-artifact@v8.0.1 + with: + name: public-cf + path: public-cf + + - name: Ensure Cloudflare Pages config + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: python3 scripts/ensure_cloudflare_pages.py --project kage --domain kage.tamnd.com --zone tamnd.com + + - name: Deploy to Cloudflare Pages + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + npx -y wrangler@4 pages deploy public-cf \ + --project-name=kage \ + --branch=main \ + --commit-dirty=true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..baaf903 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,112 @@ +name: release + +# Pushing a version tag turns the GoReleaser config (.goreleaser.yaml) into a +# GitHub release with the archives, Linux packages (deb, rpm, apk), checksums, +# SBOMs and a cosign signature, and pushes the multi-arch container image to +# GHCR. Pull requests and pushes to main run `goreleaser check` only, so a +# config that would fail a real release is caught long before the tag. + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Fast gate on every PR and push: the config parses and is valid for the + # installed GoReleaser version. No artifacts are built here. + check: + if: github.ref_type != 'tag' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + with: + fetch-depth: 0 + - uses: actions/setup-go@v6.4.0 + with: + go-version-file: go.mod + check-latest: true + cache: true + - uses: goreleaser/goreleaser-action@v7.2.2 + with: + distribution: goreleaser + version: "~> v2" + args: check + + # The real release, only on a version tag. + release: + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + contents: write # create the GitHub release + packages: write # push the image to ghcr.io + id-token: write # keyless cosign signing + env: + LINUX_REPO_DISPATCH_TOKEN: ${{ secrets.LINUX_REPO_DISPATCH_TOKEN }} + steps: + - uses: actions/checkout@v7.0.0 + with: + fetch-depth: 0 + - uses: actions/setup-go@v6.4.0 + with: + go-version-file: go.mod + check-latest: true + cache: true + + # Build and ship the linux/arm64 image from the amd64 runner. + - uses: docker/setup-qemu-action@v4.1.0 + - uses: docker/setup-buildx-action@v4.1.0 + - uses: docker/login-action@v4.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Tools GoReleaser shells out to for signing and SBOMs. + # Pin cosign to the 2.x line. cosign 3.x makes the new bundle format the + # default, which ignores the --output-signature/--output-certificate flags + # the signs block uses and aborts. Pinning keeps the .sig/.pem outputs and + # stops the release tool from floating to a breaking latest. + - uses: sigstore/cosign-installer@v4.1.2 + with: + cosign-release: "v2.6.3" + - uses: anchore/sbom-action/download-syft@v0.24.0 + + - uses: goreleaser/goreleaser-action@v7.2.2 + with: + distribution: goreleaser + version: "~> v2" + args: release --clean + env: + # Creating the release and pushing the GHCR image use the built-in + # token. The package-manager publish steps each read their own secret; + # any that is unset leaves that manager skipped (the artifact is still + # produced), so the release never fails for a tap that is not set up. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} + SCOOP_BUCKET_GITHUB_TOKEN: ${{ secrets.SCOOP_BUCKET_GITHUB_TOKEN }} + + # Rebuild the Linux apt/dnf repository with the packages just released. + # Skipped when the dispatch token is unset, so the release never depends + # on it being configured. + - name: Refresh the Linux package repository + if: env.LINUX_REPO_DISPATCH_TOKEN != '' + run: | + code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + -H "Authorization: Bearer $LINUX_REPO_DISPATCH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/tamnd/linux-repo/dispatches \ + -d '{"event_type":"package-released"}' || true) + if [ "$code" = "204" ]; then + echo "Linux repo refresh triggered" + else + echo "Linux repo refresh not accepted (HTTP $code); skipping" + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d7b20e6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +/bin/ +/dist/ +/kage-out/ +*.out +.DS_Store + +# Built docs site (tago output) and its cache +/docs/public/ +/docs/public-pages/ +/docs/public-cf/ +**/.tago-cache.db diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d2f84c4 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "docs/themes/tago-doks"] + path = docs/themes/tago-doks + url = https://github.com/tamnd/tago-doks diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..a4b49ce --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,247 @@ +# GoReleaser turns one tag push into everything a user might install from: raw +# archives, Linux packages (deb, rpm, apk), a multi-arch container image, and +# entries for the package managers (Homebrew, Scoop). `git tag vX.Y.Z && git +# push --tags` fans out to all of them through .github/workflows/release.yml. +# +# Publish steps that push to a repository we do not own yet (the Homebrew tap, +# the Scoop bucket) self-disable when their token is absent. A release with no +# extra secrets still produces every downloadable artifact and the container +# image; each manager lights up the moment its repository and token exist. +version: 2 + +project_name: kage + +before: + # Only fetch modules; never `go mod tidy` during a release, the tree is kept + # tidy in CI. + hooks: + - go mod download + +builds: + - id: kage + binary: kage + main: ./cmd/kage + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w + - -X github.com/tamnd/kage/cli.Version={{ .Version }} + - -X github.com/tamnd/kage/cli.Commit={{ .ShortCommit }} + - -X github.com/tamnd/kage/cli.Date={{ .CommitDate }} + mod_timestamp: "{{ .CommitTimestamp }}" + targets: + - linux_amd64 + - linux_arm64 + - linux_arm_7 + - linux_386 + - darwin_amd64 + - darwin_arm64 + - windows_amd64 + - windows_arm64 + - freebsd_amd64 + - freebsd_arm64 + + # A second Windows build linked for the GUI subsystem (-H windowsgui). It is + # the same pure-Go kage, but a viewer packed onto it shows only its window when + # double-clicked, with no console flashing behind it. Users point `kage pack + # --base` at this to build a clean double-click Windows app. The default build + # above stays console-attached so the CLI still prints clone progress. + - id: kage-gui + binary: kage + main: ./cmd/kage + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w + - -H=windowsgui + - -X github.com/tamnd/kage/cli.Version={{ .Version }} + - -X github.com/tamnd/kage/cli.Commit={{ .ShortCommit }} + - -X github.com/tamnd/kage/cli.Date={{ .CommitDate }} + mod_timestamp: "{{ .CommitTimestamp }}" + targets: + - windows_amd64 + - windows_arm64 + +archives: + # tar.gz everywhere except a zip on Windows. Scoped to the console build so + # this is the archive every package manager installs. + - id: default + ids: + - kage + name_template: "kage_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}" + format_overrides: + - goos: windows + formats: [zip] + files: + - LICENSE + - README.md + + # A separate zip for the GUI-subsystem Windows binary, named so it cannot be + # confused with the console build. It is a base to pack against, not something + # to run directly, so no package manager points at it. + - id: windows-gui + ids: + - kage-gui + name_template: "kage_{{ .Version }}_windows-gui_{{ .Arch }}" + formats: [zip] + files: + - LICENSE + - README.md + +nfpms: + # One nfpm definition emits the deb, rpm, and apk for every Linux build. kage + # is a user command, not a daemon, so there is no unit file and no + # postinstall: the package is the binary and its license. Chrome is a runtime + # dependency the user supplies (or the container image bundles). + - id: linux-packages + package_name: kage + ids: + - kage + file_name_template: "{{ .ConventionalFileName }}" + vendor: tamnd + homepage: https://github.com/tamnd/kage + maintainer: Duc-Tam Nguyen + description: Clone any website for offline viewing, with the JavaScript stripped out. + license: MIT + formats: + - deb + - rpm + - apk + bindir: /usr/bin + section: utils + recommends: + - chromium + contents: + - src: ./LICENSE + dst: /usr/share/doc/kage/LICENSE + +dockers_v2: + # One multi-platform image built with buildx. GoReleaser stages the prebuilt + # binaries under per-platform directories in the build context and the + # Dockerfile copies the right one through $TARGETPLATFORM. + - images: + - ghcr.io/tamnd/kage + tags: + - "{{ .Version }}" + - latest + dockerfile: Dockerfile + platforms: + - linux/amd64 + - linux/arm64 + labels: + org.opencontainers.image.title: "{{ .ProjectName }}" + org.opencontainers.image.description: "Clone any website for offline viewing, with the JavaScript stripped out" + org.opencontainers.image.url: "https://github.com/tamnd/kage" + org.opencontainers.image.source: "https://github.com/tamnd/kage" + org.opencontainers.image.version: "{{ .Version }}" + org.opencontainers.image.revision: "{{ .FullCommit }}" + org.opencontainers.image.licenses: "MIT" + +homebrew_casks: + # Homebrew cask pushed to the tap repository. Self-disables until + # HOMEBREW_TAP_GITHUB_TOKEN (a PAT with write to tamnd/homebrew-tap) is set, + # so a tokenless release still writes the cask into dist for inspection. + - name: kage + ids: + - default + repository: + owner: tamnd + name: homebrew-tap + token: '{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}' + directory: Casks + homepage: https://github.com/tamnd/kage + description: Clone any website for offline viewing, with the JavaScript stripped out + skip_upload: '{{ if index .Env "HOMEBREW_TAP_GITHUB_TOKEN" }}false{{ else }}true{{ end }}' + commit_author: + name: Duc-Tam Nguyen + email: tamnd87@gmail.com + # Homebrew quarantines cask artifacts, and Gatekeeper kills quarantined + # binaries that are only ad-hoc signed (which cross-compiled Go binaries + # are). Strip the attribute at install so the binary runs first try. + hooks: + post: + install: | + if system_command("/usr/bin/xattr", args: ["-h"]).exit_status.zero? + system_command "/usr/bin/xattr", args: ["-dr", "com.apple.quarantine", "#{staged_path}/kage"] + end + +scoops: + # Scoop manifest for Windows, pushed to the bucket repository. It installs the + # console build (the CLI), not the GUI base. + - ids: + - default + repository: + owner: tamnd + name: scoop-bucket + token: '{{ .Env.SCOOP_BUCKET_GITHUB_TOKEN }}' + directory: bucket + homepage: https://github.com/tamnd/kage + description: Clone any website for offline viewing, with the JavaScript stripped out + license: MIT + skip_upload: '{{ if index .Env "SCOOP_BUCKET_GITHUB_TOKEN" }}false{{ else }}true{{ end }}' + commit_author: + name: Duc-Tam Nguyen + email: tamnd87@gmail.com + +checksum: + name_template: "checksums.txt" + algorithm: sha256 + +sboms: + # A CycloneDX SBOM per archive via syft; the release workflow installs it. + - id: archive + artifacts: archive + +signs: + # Keyless cosign signature over the checksum file. It runs only on a real + # release in CI, where the workflow grants the OIDC token cosign needs. + - cmd: cosign + certificate: "${artifact}.pem" + args: + - sign-blob + - "--output-certificate=${certificate}" + - "--output-signature=${signature}" + - "${artifact}" + - "--yes" + artifacts: checksum + output: true + +docker_signs: + - cmd: cosign + artifacts: manifests + args: + - sign + - "${artifact}@${digest}" + - "--yes" + +changelog: + sort: asc + use: github + filters: + exclude: + - "^docs:" + - "^test:" + - "^chore:" + - "^ci:" + - Merge pull request + - Merge branch + groups: + - title: Features + regexp: '^.*?feat(\(.+\))??!?:.+$' + order: 0 + - title: Fixes + regexp: '^.*?fix(\(.+\))??!?:.+$' + order: 1 + - title: Other + order: 999 + +release: + github: + owner: tamnd + name: kage + draft: false + prerelease: auto diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..07f7992 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,278 @@ +# Changelog + +All notable changes to kage are recorded here. The format follows +[Keep a Changelog](https://keepachangelog.com/), and the project aims to follow +[Semantic Versioning](https://semver.org/). + +## [Unreleased] + +## [0.3.9] - 2026-07-08 + +### Fixed + +- The Windows build no longer embeds the leakless watchdog binary that Windows Defender flags as `Trojan:Win32/Kepavll!rfn`, which made a fresh `scoop install` fail with a virus warning on `leakless.exe` ([#68](https://github.com/tamnd/kage/issues/68)). + go-rod's launcher imports [leakless](https://github.com/ysmood/leakless), which base64/gzip-embeds a prebuilt helper for every platform and links the Windows one into `kage.exe`. + kage already launches Chrome with leakless disabled, so the helper never ran, only added the flagged bytes. + A `replace` directive now points the package at an API-compatible stub under `third_party/leakless` that carries no embedded binary, dropping about 1.28 MB from the Windows build. + +## [0.3.6] - 2026-06-19 + +### Fixed + +- `kage clone --mobile` now produces a correct layout when the source page uses table-based navigation with image maps, as paulgraham.com does. + Three rendering problems appeared in Kiwix iOS after the 0.3.5 release: a tall blank box at the top of every page, content clipped on the right edge, and a narrow spacer column occupying screen space. + The blank box came from hiding only the `` element while leaving its `` container in place; the cell collapsed to empty but kept its full height. + `td:has(>img[usemap])` now hides the entire nav column rather than just the image inside it. + The right-side clip came from `width="435"` set directly as an HTML attribute on the inner content table; the earlier `max-width` CSS rule does not override HTML attributes. + A new `[width]{width:auto!important;max-width:100%!important}` rule cancels every fixed HTML width attribute on any element — tables, cells, and images alike — so the content column stretches to fill the phone screen. + The spacer column (a 26 px `` holding a 1×1 transparent GIF) kept its allocated width for the same reason; `td:has(>img[src*="trans_1x1"]:only-child)` hides it alongside the nav column. + Two additional rules round out the fix: `*{box-sizing:border-box}` prevents padding from pushing content past the viewport edge, and `img{max-width:100%;height:auto}` keeps any inline photos within their column. + +## [0.3.5] - 2026-06-19 + +### Changed + +- Each saved page is now stored in a packed ZIM under its own `` instead of its URL path, so a ZIM reader's search box suggests pages by their readable title. + Typing into Kiwix's search now offers "Five Founders" or "Female Founders" rather than a filename, because the title pointer list a reader's suggestion search walks carries the real page titles. + A page with no `<title>` still falls back to its path, and the per-page title also flows into the `title` column of `kage parquet export`. + +### Added + +- `kage clone --mobile` makes legacy "font-era" sites readable on a phone. + Sites from the 1990s and early 2000s — paulgraham.com is a good example — embed typography directly in the HTML with `<font size="2" face="verdana">`, table-based layouts, and no viewport declaration. + A mobile browser receiving that markup without a viewport meta shrinks everything to desktop scale, and the `<font size="2">` instruction then makes the already-small text microscopic. + Passing `--mobile` injects two things into every saved page before it is written: a `<meta name="viewport" content="width=device-width, initial-scale=1">` tag so the browser stops shrinking, and a small `<style>` block that lifts the base font size to 18 px, inherits that size through `<font>` elements, caps the content width at 720 px, loosens line height to 1.7, and hides image-map navigation elements (usually a GIF served from an external CDN that 404s offline anyway). + The override is deliberately last in `<head>` so it wins specificity ties, and it does not touch pages that already carry a viewport and readable type sizes. +- The packing guide now documents how search works on a kage archive: title suggestions in any ZIM reader, and full-text search of page bodies through `kage parquet export` and DuckDB. + A Xapian full-text index is deliberately not written, since Xapian is GPL and kage is MIT. + +## [0.3.4] - 2026-06-17 + +### Fixed + +- `kage serve` now stops on Ctrl-C instead of ignoring it. + The preview server was started with a blocking call that never watched for an interrupt, so the only way to stop it was to kill the process. + kage now shuts the server down gracefully on an interrupt or a `SIGTERM`, with a short timeout before it forces the listener closed, so a preview exits cleanly. Thanks to Xirui Wang (#35) and Kaidi Zhao (#38). +- A page whose JavaScript builds a deeply nested object graph no longer fails to clone. + Chrome's DevTools Protocol returns "Object reference chain is too long" while loading such a page, but the HTML has already loaded and the error is only about Chrome's internal object tracking, not the document. + kage now recognises that specific error and finishes rendering the page instead of dropping it (reported in #36). Thanks to Gautam Kumar (#39). + +## [0.3.3] - 2026-06-16 + +### Fixed + +- Chrome no longer downloads a file to your Downloads folder when a crawl follows a link that turns out to be a binary (reported in #32). + An extensionless link is queued as a page, so the page worker navigated to it in Chrome, and a link that served a zip or a CSV made Chrome save the file to `~/Downloads`, a surprise side effect of a clone. + kage now denies Chrome-initiated downloads browser-wide, since every asset is fetched through kage's own downloader, and detects a navigation whose response is not HTML and reroutes that URL to the asset downloader, where the size and media policy decides whether to localise it or leave it on the live web. + +## [0.3.2] - 2026-06-16 + +### Fixed + +- Saved pages now declare their character encoding, so text no longer mojibakes in a reader. + kage writes every page as UTF-8, but a source that set its charset only in the HTTP `Content-Type` header, with no `<meta charset>` in the markup, lost that signal once the page became a standalone file. + A reader serving the bytes without a charset then fell back to its locale encoding and turned every curly quote, dash, and non-breaking space into mojibake (reported in #16 and #29). + kage now inserts a `<meta charset="utf-8">` at the top of `<head>` when the page does not already declare one, so the page is self-describing in any reader. + +## [0.3.1] - 2026-06-15 + +### Fixed + +- A served mirror whose entry point is a nested page no longer loses its CSS and + images when opened at the root. kage saves each page's asset links as + mirror-relative paths (`../_kage/...`) computed for that page's own location, + but the viewer answered `/` with the main page's bytes in place, so the browser + resolved those relative URLs against `/` and missed every one. A + `developer.apple.com/documentation` mirror, whose main page is + `developer.apple.com/documentation/index.html`, landed at `/` completely + unstyled. kage now redirects `/` to the main page's canonical content path, the + way the archive's `W/mainPage` redirect already does, so the browser resolves + the page's relative assets correctly. Kiwix was unaffected because it follows + that redirect itself. + +## [0.3.0] - 2026-06-15 + +### Added + +- `kage parquet export <file.zim>` and `kage parquet import <file.parquet>` + convert a packed archive to a columnar Parquet table and back. The table is + flat, one row per archive entry, with clear columns (url, host, title, mime, + extracted text, content), so it drops straight into the tooling a dataset host + like Hugging Face expects, and DuckDB or pandas can query it as is. The column + names follow the open-index/open-markdown dataset (`doc_id`, `url`, `host`, + `crawl_date`, `content_length`, `text_length`, `text`), with `doc_id` a + deterministic UUID v5 of the page URL, so a kage export sits alongside other + web-crawl datasets. The conversion is lossless: a ZIM round-tripped through + Parquet reproduces every entry, its metadata, and the main page byte for byte. +- `kage pack --incremental` keeps a small cache sidecar next to the output and + reuses the compression of any cluster whose bytes have not changed since the + last pack. Compressing clusters with zstd is the dominant cost of packing a + large mirror, so re-packing after a small change (a `--refresh`, a handful of + edited pages) only compresses what actually changed instead of the whole + archive. A cached cluster is byte-for-byte what a fresh compression produces, + so the archive stays deterministic and valid. The pack reports how many + clusters it reused versus compressed. +- Identical pages are now stored once. When a rendered page's bytes match a page + already written, kage stores it as a hard link to the first copy instead of a + second full file. This collapses the duplicate content a faceted site spawns + when many `?q=…`/`?page=…` URLs all render the same page. The final summary + reports how many pages were deduped this way. + +### Changed + +- Clones no longer pull a site's bulk downloads into the mirror by default. Video + and audio, installers and disk images (`.dmg`, `.pkg`, `.exe`, `.msi`, ...), + archives, and PDFs are left pointing at their live URL instead of downloaded, + because they are rarely needed to read a site offline yet routinely make up + most of its bytes (a developer.apple.com crawl was 18 of 19 GB of such assets). + Page-rendering assets (images, fonts, CSS) are unaffected. `--keep-media` + restores the old behavior, and `--skip-ext .foo` leaves more extensions remote. +- Assets are localized only from the seed's own registrable domain by default. + developer.apple.com still pulls from www.apple.com and images.apple.com, but a + separate brand domain or an unrelated third party (an embedded tracker, an + off-topic CDN) is left on the live web rather than mirrored. `--all-asset-hosts` + restores downloading assets from any host. +- The progress line now counts real pages. "pages" is the number of distinct URL + paths written, and the query-string variants that one path can spawn by the + thousand are shown separately as "variants", so the live counter tracks the + site's real size instead of being inflated by `?q=…` permutations. + +### Fixed + +- An asset larger than the size cap (`--max-asset-mb`, 25 by default) is now + skipped instead of being truncated to a corrupt fragment. The cap was a + `LimitReader`, so an over-size file was saved as exactly the first N MB of + itself: a broken video or installer that wasted disk and would never play or + run. kage now checks the response size and leaves an over-cap asset out of the + mirror entirely, pointing at its live URL. On a developer.apple.com crawl this + was around a gigabyte of truncated WWDC videos and `.dmg` installers. +- An asset URL whose query string carries a raw space is now requested with the + space percent-encoded, so the server gets a valid request instead of rejecting + it. Real sites bust a stylesheet's cache with a date, producing an href like + `styles/main.css?Thursday, 26-Feb-2026 16:26:41 UTC`; a browser encodes the + spaces before requesting, but kage was passing them through verbatim and the + server answered `400 Bad Request`. On a developer.apple.com crawl this was the + cause of the large majority of the download errors. The query is re-encoded on + the canonical URL, so the on-disk key matches the fixed request. + +## [0.2.1] - 2026-06-15 + +### Added + +- ZIM archives now carry the metadata Kiwix and `zimcheck` treat as mandatory. Every archive gets a `Name` and a `Description` (a host-derived line when `--description` is not given), and, when the mirror has a usable favicon, an `Illustrator_48x48@1` entry: the icon rescaled to a 48x48 PNG, which is the book icon Kiwix shows in its library. + +## [0.2.0] - 2026-06-15 + +### Added + +- `kage pack --app` wraps the packed viewer in a double-click desktop app with + the site's favicon as the icon. The flag builds on the binary format, so it + composes with `--base` (including a `webview` base) and `--icon`. On macOS it + writes a `.app` bundle (`Info.plist`, the viewer under `Contents/MacOS`, and an + `.icns` generated from the favicon); on Linux, with a Linux `--base`, it writes + an AppImage-style `.AppDir` and folds it into a single `.AppImage` when + `appimagetool` is installed. The icon is found in the mirror automatically + (preferring a large `apple-touch-icon.png`, then `favicon.png` or a PNG-based + `favicon.ico`) and can be overridden with `--icon`. +- The release now ships a GUI-subsystem Windows base, + `kage_<version>_windows-gui_<arch>.zip`. Packing a viewer onto it with + `--format binary --base` produces a `.exe` that opens with no console window + behind it, the Windows equivalent of the `.app` double-click experience. + +### Changed + +- Cross-platform packing detects the base binary's target OS from its executable + header (ELF, PE, or Mach-O) rather than its file name, so a Windows viewer + always gets a `.exe` suffix and the run hint names the right platform even when + the base is named without one. + +## [0.1.2] - 2026-06-15 + +### Security + +- Chrome now keeps its sandbox on by default. It was previously launched with `--no-sandbox` unconditionally, which removed Chrome's main line of defense when rendering pages from the open web (reported in #10). The sandbox is now dropped only where it genuinely cannot run: inside a container, or when running as root, and the choice is logged so it is never silent. + +### Added + +- Container-aware Chrome flags. kage detects a container from the `IN_DOCKER` environment variable or a `/.dockerenv` marker and, only there, drops the sandbox and adds `--disable-dev-shm-usage` (the default 64 MB `/dev/shm` is too small for Chrome on large pages). Outside a container the faster shared memory is left in place. +- Asset downloads retry on a transient failure (a 403/429, a 5xx, or a network blip) with a short backoff, recovering files that bot-protection rejects on the first request of a burst. Permanent failures (404, 401, ...) are not retried. + +### Changed + +- Clearer crawl error reporting. Each failure is logged with a classified reason (`HTTP 403 Forbidden`, `timed out`, ...), the URL, and the page that referenced it, and the end-of-run summary lists what went wrong instead of printing only a count. + +### Fixed + +- The container image now runs. Chrome aborted on launch with `chrome_crashpad_handler: --database is required`, so kage disables Chrome's crash reporter inside a container, and the `kage` user now has a writable home (the mounted `/out` volume) so the default output, resume state, and Chrome's profile no longer fail with a permission error (issue #7). + +## [0.1.1] - 2026-06-14 + +### Added + +- `kage pack <mirror-dir>` packs a cloned folder into one distributable file. + `--format zim` (the default) writes an open ZIM archive, the same single-file + format Kiwix uses; `--format binary` appends that archive to a copy of kage to + produce a self-contained executable that serves the site offline when run. + Flags cover the output path, metadata (`--title`, `--description`, + `--language`, `--date`), a `--base` binary for cross-platform viewers, and + `--no-compress`. +- `kage open <file.zim>` serves a packed ZIM over a local HTTP server and opens + your browser, the read side of `kage pack --format zim`. +- An optional native-window viewer. Built with `-tags webview` (which needs + cgo), `kage open` and a packed binary present the offline site in a real + window backed by the operating system's WebView (WKWebView, WebView2, + WebKitGTK) instead of a browser tab, so a packed kage feels like a standalone + app. The default build stays pure Go (`CGO_ENABLED=0`) and falls back to the + system browser, so the release pipeline is unchanged. +- A pure-Go `zim` package that writes and reads the ZIM format: a fixed header, + MIME and pointer lists, zstd-compressed (or stored) clusters, redirects, and a + trailing MD5. It reads xz clusters so archives from other tooling open, and + writes zstd or stored only. Packing is deterministic: the same mirror produces + a byte-identical archive, with the UUID derived from the content rather than + randomised. + +## [0.1.0] - 2026-06-14 + +The first release. kage clones a live website into a self-contained folder you +can browse offline, with every script stripped out. + +### Added + +- `kage clone <url>` renders each page in headless Chrome, snapshots the final + DOM, removes every `<script>`, `on*` handler, and `javascript:` URL, and + downloads the CSS, images, fonts, and media, rewriting them to local paths. +- `kage serve [dir]` runs a local static file server over a cloned folder so the + mirror's links and assets resolve the way they would on a real host. +- Deterministic URL-to-path mapping: pages become `<slug>/index.html` + directories, assets live under the reserved `_kage/<host>/` tree, and query + strings fold into a short hash suffix so versioned URLs never collide. +- Three concurrency tiers run in parallel: page-render workers (`--workers`), + asset-download workers (`--asset-workers`), and a Chrome page pool + (`--browser-pages`). +- A polite crawl by default: honours `robots.txt`, seeds from `sitemap.xml`, + and scopes to the seed host. `--scope-prefix`, `--max-depth`, `--max-pages`, + `--subdomains`, and `--exclude` shape the frontier. +- Idempotent, resumable crawling. Each page is keyed by the file it writes, so + the same URL reached over http and https, with or without a trailing slash, + or as `/index.html` versus `/`, is fetched exactly once. A re-run resumes from + `_kage/state.json`; `--refresh` re-renders a mirror in place to pull in + changed content; `--force` wipes and starts clean; `--no-resume` runs + stateless. +- Defaults to a per-user data directory (`$HOME/data/kage`), overridable with + `-o/--out`. +- Cross-platform distribution: prebuilt archives, `.deb`/`.rpm`/`.apk` packages, + a multi-arch container image on GHCR (Chromium bundled), checksums, SBOMs, and + a cosign signature, all cut from one version tag by GoReleaser. + +[Unreleased]: https://github.com/tamnd/kage/compare/v0.3.9...HEAD +[0.3.9]: https://github.com/tamnd/kage/compare/v0.3.8...v0.3.9 +[0.3.4]: https://github.com/tamnd/kage/compare/v0.3.3...v0.3.4 +[0.3.3]: https://github.com/tamnd/kage/compare/v0.3.2...v0.3.3 +[0.3.2]: https://github.com/tamnd/kage/compare/v0.3.1...v0.3.2 +[0.3.1]: https://github.com/tamnd/kage/compare/v0.3.0...v0.3.1 +[0.3.0]: https://github.com/tamnd/kage/compare/v0.2.1...v0.3.0 +[0.2.1]: https://github.com/tamnd/kage/compare/v0.2.0...v0.2.1 +[0.2.0]: https://github.com/tamnd/kage/compare/v0.1.2...v0.2.0 +[0.1.2]: https://github.com/tamnd/kage/compare/v0.1.1...v0.1.2 +[0.1.1]: https://github.com/tamnd/kage/compare/v0.1.0...v0.1.1 +[0.1.0]: https://github.com/tamnd/kage/releases/tag/v0.1.0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2cc0ed5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +# Consumed by GoReleaser: it copies the already cross-compiled binary out of the +# build context rather than compiling, so the image build is fast and uses the +# same static binary every other artifact ships. +# +# kage always drives a real headless Chrome, so unlike a plain CLI image this one +# bundles Chromium. KAGE_CHROME points kage at the system binary so it never +# tries to download its own. +# +# GoReleaser builds one multi-platform image with buildx and stages each +# platform's binary under a $TARGETPLATFORM directory (e.g. linux/amd64/) in the +# build context, so the COPY line selects the right one through the automatic +# TARGETPLATFORM build arg. +FROM alpine:3.21 + +ARG TARGETPLATFORM + +# chromium for rendering; ca-certificates for HTTPS; tzdata for sane timestamps; +# the font package so rendered pages have glyphs to lay out. +RUN apk add --no-cache chromium ca-certificates tzdata font-noto \ + && mkdir -p /out + +COPY $TARGETPLATFORM/kage /usr/bin/kage + +WORKDIR /out + +# Point kage at the bundled Chromium and write mirrors under /out by default: +# +# docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com +# +# The container runs as root, and that is deliberate (issue #7). A bind-mounted +# /out is owned by whoever created it on the host, so only root can reliably +# write into it; a fixed non-root uid cannot, and both kage's output and resume +# state (under $HOME/data/kage) then fail with "mkdir /out: permission denied". +# The same unwritable HOME also breaks Chrome: it launches chrome_crashpad_handler +# with an empty crash database path, which aborts the whole browser with +# "chrome_crashpad_handler: --database is required" and fails every render. +# Running as root keeps /out and HOME writable whatever the host owns, so the +# one-liner above just works. This costs nothing in the sandbox: Chrome's sandbox +# is already off inside any container (kage drops it on container detection), so +# root here does not loosen a boundary that was holding. HOME points at /out so +# the default output and Chrome's writable state both land in the mounted volume. +ENV KAGE_CHROME=/usr/bin/chromium-browser \ + HOME=/out + +VOLUME ["/out"] + +ENTRYPOINT ["/usr/bin/kage"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7679c4b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Duc-Tam Nguyen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..529153f --- /dev/null +++ b/Makefile @@ -0,0 +1,45 @@ +BIN := kage +PKG := ./cmd/kage +VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo none) +DATE := $(shell date -u +%Y-%m-%dT%H:%M:%SZ) +LDFLAGS := -s -w \ + -X github.com/tamnd/kage/cli.Version=$(VERSION) \ + -X github.com/tamnd/kage/cli.Commit=$(COMMIT) \ + -X github.com/tamnd/kage/cli.Date=$(DATE) + +export CGO_ENABLED := 0 + +.PHONY: build build-webview install test test-short vet tidy clean run + +build: + go build -ldflags "$(LDFLAGS)" -o bin/$(BIN) $(PKG) + +# A native-window viewer: opens packed sites in their own OS WebView window +# instead of the browser. Needs cgo, so it is built separately from the default +# pure-Go binary and the release pipeline. +build-webview: + CGO_ENABLED=1 go build -tags webview -ldflags "$(LDFLAGS)" -o bin/$(BIN) $(PKG) + +install: + go install -ldflags "$(LDFLAGS)" $(PKG) + +# Full suite, including the Chrome-driven end-to-end tests. +test: + go test -race ./... + +# Skip the tests that launch a real browser (CI without Chrome, quick loops). +test-short: + go test -short ./... + +vet: + go vet ./... + +tidy: + go mod tidy + +clean: + rm -rf bin + +run: build + ./bin/$(BIN) diff --git a/README.md b/README.md new file mode 100644 index 0000000..76195d2 --- /dev/null +++ b/README.md @@ -0,0 +1,284 @@ +# kage + +[![ci](https://github.com/tamnd/kage/actions/workflows/ci.yml/badge.svg)](https://github.com/tamnd/kage/actions/workflows/ci.yml) +[![Release](https://img.shields.io/github/v/release/tamnd/kage)](https://github.com/tamnd/kage/releases/latest) +[![Go Reference](https://pkg.go.dev/badge/github.com/tamnd/kage.svg)](https://pkg.go.dev/github.com/tamnd/kage) +[![Go Report Card](https://goreportcard.com/badge/github.com/tamnd/kage)](https://goreportcard.com/report/github.com/tamnd/kage) +[![License](https://img.shields.io/github/license/tamnd/kage)](./LICENSE) + +**kage** (影, "shadow") clones a website into a folder you can browse offline, with every script stripped out. It opens each page in real headless Chrome, waits for the page to settle, snapshots the DOM a human would have seen, then deletes all the JavaScript and pulls the CSS, images, and fonts down to local paths. What lands on disk looks like the live site and runs no code. + +[Install](#install) • [Quick start](#quick-start) • [Commands](#commands) • [Clone](#clone) • [Pack](#pack-it-into-one-file) • [Double-click app](#a-double-click-app) • [Native window](#a-real-window-not-a-browser-tab) • [How it works](#how-it-works) + +![kage cloning paulgraham.com, packing it into one file, and serving it back offline](docs/static/demo.gif) + +You already know the problem. You hit "Save As" on a page you want to keep, and six months later you open it to find a blank screen, a spinner that never stops, or a copy that still tries to phone home to an analytics server that no longer exists. The page was never really yours. It was a thin client for someone else's JavaScript. + +kage takes the other road. It drives a real browser, lets the page finish doing whatever it does, grabs the finished result, and then rips every script out of it. No tracking, no network calls, no surprises. Just `.html` files you can open straight off disk, hand to a friend, or pack into a single file and forget about for a decade. + +Full docs and guides live at **[kage.tamnd.com](https://kage.tamnd.com)**. + +## Install + +```bash +go install github.com/tamnd/kage/cmd/kage@latest +``` + +Prefer a prebuilt binary? Grab an archive, a `.deb`/`.rpm`/`.apk`, or a checksum from [releases](https://github.com/tamnd/kage/releases). Or let a package manager handle it: + +```bash +# Homebrew (macOS) +brew install --cask tamnd/tap/kage + +# Scoop (Windows) +scoop bucket add tamnd https://github.com/tamnd/scoop-bucket +scoop install kage + +# apt (Debian, Ubuntu) +curl -fsSL https://tamnd.github.io/linux-repo/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg +echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" | sudo tee /etc/apt/sources.list.d/tamnd.list +sudo apt update && sudo apt install kage + +# dnf (Fedora, RHEL) +sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo +sudo dnf install kage +``` + +Or skip installing Chrome yourself and use the container image, which bundles Chromium: + +```bash +docker run --rm -v "$PWD/out:/out" ghcr.io/tamnd/kage clone paulgraham.com +``` + +kage drives a real browser, so it needs Chrome or Chromium on the host. It finds a system install on its own; point it somewhere specific with `--chrome` or the `KAGE_CHROME` environment variable. The container needs nothing extra. + +Shell completion ships in the box: `kage completion bash|zsh|fish|powershell`. + +## Quick start + +Let's mirror Paul Graham's essays so you can read them on a plane, on a laptop with no wifi, or in the year 2050 after the site has finally changed its design: + +```bash +# 1. Clone the site into $HOME/data/kage/paulgraham.com/ +kage clone paulgraham.com + +# 2. Read it back offline in your browser +kage serve $HOME/data/kage/paulgraham.com +# open http://127.0.0.1:8800 +``` + +That's the whole loop. Every essay, every image, every stylesheet, frozen on your disk and runnable with zero network. The next two steps are optional but nice: collapse the whole thing into one file, and pop it open in its own window. + +```bash +# 3. Squeeze the mirror into a single shareable file +kage pack paulgraham.com # -> paulgraham.com.zim +kage open paulgraham.com.zim + +# 4. Or into one executable that *is* the site +kage pack paulgraham.com --format binary -o paulgraham +./paulgraham # serves itself, needs nothing installed +``` + +## Commands + +| Command | What it does | +| --- | --- | +| `kage clone <url>` | render a site in headless Chrome and write a browsable, script-free mirror | +| `kage serve [dir]` | preview a cloned folder over a local HTTP server | +| `kage pack <mirror-dir>` | collapse a mirror into one ZIM archive, a self-contained viewer binary, or a double-click app | +| `kage open <file.zim>` | serve a packed ZIM back for offline reading | + +## Clone + +```bash +# The whole site, into $HOME/data/kage/<host>/ +kage clone https://paulgraham.com + +# Just the first 50 pages, two links deep, for a quick taste +kage clone paulgraham.com --max-pages 50 --max-depth 2 + +# Only one section of a bigger site +kage clone go.dev --scope-prefix /doc + +# Pull in subdomains too, and scroll each page to trip lazy-loaded images +kage clone example.com --subdomains --scroll + +# Come back next month and re-render in place to catch new essays +kage clone paulgraham.com --refresh +``` + +A clone is a polite, breadth-first crawl. It reads `robots.txt`, seeds itself from `sitemap.xml`, and stays on the seed host unless you tell it otherwise. It is also stubbornly idempotent: each page is keyed by the file it writes, so the same essay reached over http and https, with or without a trailing slash, gets fetched exactly once. Hit Ctrl-C and it saves its place on the way out; run it again and it picks up where it stopped. `--refresh` re-renders in place, `--force` wipes the host and starts clean. + +The flags you'll actually reach for: + +| Flag | Default | Meaning | +|------|---------|---------| +| `-o, --out` | `$HOME/data/kage` | Output root; the mirror lands in `<out>/<host>/` | +| `-p, --max-pages` | `0` | Stop after N pages (0 = no limit) | +| `-d, --max-depth` | `0` | How many links deep to follow (0 = no limit) | +| `--scope-prefix` | | Only crawl paths starting with this prefix | +| `--subdomains` | `false` | Treat subdomains of the seed host as in scope | +| `--exclude` | | Path prefixes to skip (repeatable) | +| `--scroll` | `false` | Auto-scroll each page to trigger lazy loading | +| `--workers` | `4` | How many pages to render at once | +| `--no-robots` | `false` | Ignore `robots.txt` (be nice) | +| `--crawl-delay` | `0s` | Override robots.txt `Crawl-delay` between page starts | +| `-f, --force` | `false` | Delete any existing mirror for the host first | +| `--chrome` | | Path to the Chrome/Chromium binary | + +`kage clone --help` has the rest, including render-timing, concurrency, and asset-size knobs. + +### Serve + +`kage serve` runs a tiny static file server over a cloned folder so links and assets resolve the way they would on a real host: + +```bash +kage serve $HOME/data/kage/paulgraham.com +# open http://127.0.0.1:8800 +``` + +## Pack it into one file + +A mirror is a folder, which is great for browsing and lousy for moving around. Copying thousands of little files is slow, and "here, have this directory" is a clumsy thing to hand someone. `kage pack` collapses the whole mirror into one artifact, and you choose the shape: an open ZIM archive, or a single executable that *is* the site. + +### A single ZIM file + +```bash +kage pack paulgraham.com # -> paulgraham.com.zim +kage open paulgraham.com.zim +``` + +ZIM is an open file format built for exactly this: a whole website (or a whole Wikipedia) squeezed into one compressed, indexed, read-only file. kage writes the entire mirror into it, text zstd-compressed and media stored as-is. It is the format behind [Kiwix](https://kiwix.org), the offline-content project people use to carry Wikipedia, Stack Overflow, and Project Gutenberg onto boats, into classrooms with no internet, and onto a phone for a long flight. Because the format is a documented standard and not a kage invention, a `paulgraham.com.zim` you make today will still open in any ZIM reader years from now. + +So you are not locked into kage. `kage open` is the quickest way back in, but the very same file works across the wider Kiwix ecosystem: + +```bash +kage open paulgraham.com.zim # read it back with kage +kiwix-serve paulgraham.com.zim # or serve it with Kiwix at http://localhost +``` + +You can also double-click the file in the [Kiwix desktop app](https://kiwix.org/en/applications/), or load it on [Kiwix for Android or iOS](https://kiwix.org/en/applications/) to read your mirror on your phone. One caveat: kage writes a structurally valid archive with the standard metadata, but it does not build the full-text search index that Kiwix's own packs ship with, so browsing and clicking work everywhere while in-reader search is limited. + +Packing is deterministic. The same mirror always produces a byte-identical file, with the archive UUID derived from the content instead of randomized, so a pack is safe to checksum and cache. A bare host name resolves against the default output directory, which is why `kage pack paulgraham.com` just works right after `kage clone paulgraham.com`. + +### A self-contained binary + +`--format binary` glues the archive onto a copy of kage and hands you a single executable that serves the site offline when you run it. Whoever you send it to needs nothing installed: not kage, not a ZIM reader, nothing. + +```bash +kage pack paulgraham.com --format binary -o paulgraham +./paulgraham +``` + +The appended archive is platform-independent; only the base executable carries the architecture. By default kage appends to itself, so you get a viewer for the machine you ran it on. Point `--base` at a kage built for another OS (grab one from a [release](https://github.com/tamnd/kage/releases); every platform ships one) to produce a viewer for that platform from your own machine. kage reads the base's executable header to figure out the target, so a Windows viewer automatically gets a `.exe` name: + +```bash +# Sitting on a Mac, build a Windows viewer +kage pack paulgraham.com --format binary --base kage-windows-amd64.exe # -> paulgraham.exe +``` + +The trade is size. The binary carries a whole kage, so it weighs around 13 MiB plus the site no matter how small the mirror is. When you only need the content, the ZIM is far leaner. + +### A double-click app + +A bare binary is great from a terminal, but double-click it in a file manager and the experience is rough: macOS opens a Terminal window behind the site, and on Windows a console flashes up next to it. Add `--app` and kage wraps the same viewer in a proper desktop app so a double-click just opens the site, no terminal, with the mirror's own favicon as the icon. + +On macOS you get a real `.app` bundle: + +```bash +kage pack paulgraham.com --app # -> paulgraham.app +open paulgraham.app # or double-click it in Finder +``` + +On Linux, point `--base` at a Linux kage and you get an [AppImage](https://appimage.org)-style `.AppDir` with a `.desktop` launcher (`Terminal=false`, so no console). If [`appimagetool`](https://github.com/AppImage/appimagetool) is installed, kage folds it into a single double-clickable `.AppImage` for you: + +```bash +kage pack paulgraham.com --app --base kage-linux-amd64 # -> paulgraham.AppDir (+ .AppImage) +``` + +kage finds the icon by digging the favicon out of the mirror (it prefers a large `apple-touch-icon.png` and falls back to `favicon.ico`); pass `--icon some.png` to override it. Pair `--app` with a `webview` base (below) and the double-click opens a native window instead of the browser, which is the full "it's an app" effect. + +Windows needs no bundle, because there a single `.exe` already is the app. The catch is the console window. The release ships a `kage_<version>_windows-gui_<arch>.zip` whose binary is linked for the GUI subsystem, so a viewer packed onto it opens with no console behind it: + +```bash +# Build a console-free Windows viewer (from any OS) +kage pack paulgraham.com --format binary --base kage-windows-gui-amd64.exe # -> paulgraham.exe +``` + +## A real window, not a browser tab + +By default a packed binary opens your system browser, which means the site shows up as yet another tab, address bar and all, next to the 47 you already have open. Build kage with the `webview` tag and it opens the site in its own window instead, backed by the operating system's WebView (WKWebView on macOS, WebView2 on Windows, WebKitGTK on Linux). Paul Graham's essays, offline, in something that looks and feels like a real app: + +![paulgraham.com served offline in a native kage window](docs/static/webview.png) + +```bash +make build-webview # or: CGO_ENABLED=1 go build -tags webview ./cmd/kage +kage pack paulgraham.com --format binary --base bin/kage -o paulgraham +./paulgraham # opens a window, no browser in sight +``` + +This build needs cgo and links the platform WebView, so it stays opt-in. The default build is pure Go (`CGO_ENABLED=0`) and the prebuilt release binaries open the browser, which keeps the cross-compiled release simple. `kage open` honours the same tag, so built with `-tags webview` it shows a ZIM in a native window too. + +## How it works + +``` +seed URL ─▶ headless Chrome ─▶ final DOM ─▶ strip JS ─▶ localise assets ─▶ disk + (render) (snapshot) (sanitize) (rewrite links) +``` + +A pool of Chrome tabs renders pages; a separate pool fetches assets over plain HTTP. Every URL maps deterministically to a local path, so links get rewritten before the asset they point at has even finished downloading. The output looks like this: + +``` +paulgraham.com/ +├── index.html # the home page, scripts stripped +├── greatwork.html # /greatwork.html, an essay +├── _kage/ # reserved: assets and crawl state +│ ├── paulgraham.com/site.css # localised stylesheet (url() rewritten) +│ ├── paulgraham.com/pg.png +│ └── state.json # visited set, for resuming +└── ... +``` + +`pack` rides on the same idea: the mirror's links are already mirror-relative paths, and those map one-to-one onto the archive's content entries, so a click in a served page hits the right entry with no rewriting at all. + +## Building from source + +```bash +git clone https://github.com/tamnd/kage +cd kage +make build # -> bin/kage (pure Go, opens the browser) +make build-webview # -> bin/kage with the native-window viewer (needs cgo) +make test # full suite, including the Chrome-driven end-to-end tests +make test-short # skip the tests that launch a browser +``` + +The repo is split by concern: + +``` +cmd/kage/ thin main: pins the main thread, then hands off to cli.Execute +cli/ the cobra command tree and flag wiring +clone/ the crawl: frontier, render workers, asset workers, resume state +browser/ headless Chrome control and DOM snapshotting +sanitize/ strip scripts, handlers, and javascript: URLs from the DOM +asset/ download and localise CSS, images, and fonts +urlx/ the deterministic URL-to-path mapping +zim/ a pure-Go ZIM reader and writer +pack/ mirror to ZIM or self-contained binary, and the offline HTTP handler +viewer/ present a served site: system browser, or native window (webview tag) +docs/ the tago documentation site +``` + +## Releasing + +Push a version tag and GitHub Actions runs GoReleaser, which builds the archives, the `.deb`/`.rpm`/`.apk` packages, a multi-arch GHCR image with Chromium bundled, checksums, SBOMs, and a cosign signature: + +```bash +git tag v0.1.1 +git push --tags +``` + +The image tag carries no `v` prefix (`ghcr.io/tamnd/kage:0.1.1`). The Homebrew and Scoop steps self-disable until their tokens exist, so the first release works with no extra secrets. + +## License + +MIT. See [LICENSE](LICENSE). diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..8ab52d9 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`frostbyte_neo/kage` +- 原始仓库：https://gitea-dev.autobee.pro/frostbyte_neo/kage +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/asset/asset_test.go b/asset/asset_test.go new file mode 100644 index 0000000..d347111 --- /dev/null +++ b/asset/asset_test.go @@ -0,0 +1,118 @@ +package asset + +import ( + "bytes" + "net/url" + "sort" + "strings" + "testing" + + "github.com/tamnd/kage/urlx" + "golang.org/x/net/html" +) + +// testSink builds a RefSink that records every URL it sees and rewrites it to a +// local relative path from fromDir, mirroring what the cloner does. +func testSink(seedHost, fromDir string, seen *[]string) RefSink { + return func(u *url.URL, kind urlx.Kind) string { + *seen = append(*seen, u.String()) + if !urlx.SameSite(&url.URL{Host: seedHost}, u, true) && kind == urlx.Page { + return u.String() // external page links stay absolute + } + local := urlx.LocalPath(seedHost, u, kind, "") + return urlx.Rel(fromDir, local) + } +} + +func TestRewriteCSS(t *testing.T) { + base, _ := url.Parse("https://ex.com/css/main.css") + in := `@import "base.css"; +@import url('https://cdn.io/reset.css'); +body { background: url(../img/bg.png); } +.x { background: url("https://cdn.io/sprite.svg"); } +.y { content: url(data:image/gif;base64,AAAA); }` + var seen []string + // CSS lives at _kage/ex.com/css/main.css → its dir. + fromDir := "_kage/ex.com/css" + out := string(RewriteCSS([]byte(in), base, testSink("ex.com", fromDir, &seen))) + + if !strings.Contains(out, `@import "base.css"`) { + t.Errorf("local @import not rewritten relative: %s", out) + } + if !strings.Contains(out, `url("../img/bg.png")`) { + t.Errorf("local url() not rewritten relative: %s", out) + } + // Cross-origin asset goes under _kage/cdn.io/...; from _kage/ex.com/css that + // is ../../cdn.io/... + if !strings.Contains(out, `url("../../cdn.io/reset.css")`) { + t.Errorf("cross-origin @import not localised: %s", out) + } + if !strings.Contains(out, `url("../../cdn.io/sprite.svg")`) { + t.Errorf("cross-origin url() not localised: %s", out) + } + // data: URL must be left exactly as-is. + if !strings.Contains(out, "url(data:image/gif;base64,AAAA)") { + t.Errorf("data: URL was altered: %s", out) + } + + sort.Strings(seen) + want := []string{ + "https://cdn.io/reset.css", + "https://cdn.io/sprite.svg", + "https://ex.com/css/base.css", + "https://ex.com/img/bg.png", + } + if strings.Join(seen, ",") != strings.Join(want, ",") { + t.Errorf("sink saw %v, want %v", seen, want) + } +} + +const htmlDoc = `<!doctype html><html><head> +<link rel="stylesheet" href="/css/main.css"> +<link rel="preload stylesheet" href="/css/vp.css" as="style"> +<link rel="icon" href="/favicon.ico"> +<link rel="canonical" href="https://ex.com/canon"> +</head><body> +<a href="/docs/intro">internal</a> +<a href="https://other.com/x">external</a> +<a href="/files/report.pdf">a pdf</a> +<img src="/img/logo.png" srcset="/img/logo.png 1x, /img/logo@2x.png 2x"> +<p style="background:url(/img/bg.png)">hi</p> +<style>.a{background:url(/img/tile.png)}</style> +</body></html>` + +func TestRewriteHTML(t *testing.T) { + base, _ := url.Parse("https://ex.com/") + root, err := html.Parse(strings.NewReader(htmlDoc)) + if err != nil { + t.Fatal(err) + } + var seen []string + // The page is the root index.html, so fromDir is "". + RewriteHTML(root, base, testSink("ex.com", "", &seen)) + + var buf bytes.Buffer + if err := html.Render(&buf, root); err != nil { + t.Fatal(err) + } + out := buf.String() + + checks := map[string]bool{ + `href="_kage/ex.com/css/main.css"`: true, // stylesheet localised + `href="_kage/ex.com/css/vp.css"`: true, // multi-value "preload stylesheet" rel localised + `href="_kage/ex.com/favicon.ico"`: true, // icon localised + `href="https://ex.com/canon"`: true, // canonical left alone + `href="docs/intro/index.html"`: true, // internal page → local + `href="https://other.com/x"`: true, // external page stays absolute + `href="_kage/ex.com/files/report.pdf"`: true, // pdf link treated as asset + `src="_kage/ex.com/img/logo.png"`: true, + `_kage/ex.com/img/logo@2x.png 2x`: true, // srcset candidate rewritten + `background:url("_kage/ex.com/img/bg.png")`: true, // inline style (attr-escaped quotes) + `background:url("_kage/ex.com/img/tile.png")`: true, // <style> text + } + for frag, want := range checks { + if strings.Contains(out, frag) != want { + t.Errorf("fragment %q present=%v, want %v\n--- output ---\n%s", frag, !want, want, out) + } + } +} diff --git a/asset/css.go b/asset/css.go new file mode 100644 index 0000000..5cd25c9 --- /dev/null +++ b/asset/css.go @@ -0,0 +1,69 @@ +package asset + +import ( + "net/url" + "regexp" + "strings" + + "github.com/tamnd/kage/urlx" +) + +// RefSink registers a resolved asset/page URL with the cloner and returns the +// string to write back into the markup or CSS — a relative local path for +// things kage saves, or the absolute URL for anything it leaves on the live web. +type RefSink func(u *url.URL, kind urlx.Kind) string + +var ( + // url( ... ) with optional single/double quotes or bare token. + cssURLRe = regexp.MustCompile(`url\(\s*("[^"]*"|'[^']*'|[^)'"]*)\s*\)`) + // @import "..." / @import '...' (the @import url(...) form is caught by cssURLRe). + cssImportRe = regexp.MustCompile(`@import\s+("[^"]*"|'[^']*')`) +) + +// RewriteCSS rewrites every url(...) and @import in a stylesheet so its +// references point at local files. base is the stylesheet's own URL (so relative +// references resolve correctly); sink maps each absolute URL to its local path. +// data: URLs and unparseable references are left untouched. +func RewriteCSS(css []byte, base *url.URL, sink RefSink) []byte { + s := string(css) + s = cssImportRe.ReplaceAllStringFunc(s, func(m string) string { + raw := cssImportRe.FindStringSubmatch(m)[1] + ref := unquote(raw) + if newRef, ok := resolveRef(base, ref, sink); ok { + return `@import "` + newRef + `"` + } + return m + }) + s = cssURLRe.ReplaceAllStringFunc(s, func(m string) string { + raw := cssURLRe.FindStringSubmatch(m)[1] + ref := unquote(raw) + if newRef, ok := resolveRef(base, ref, sink); ok { + return `url("` + newRef + `")` + } + return m + }) + return []byte(s) +} + +// resolveRef normalises ref against base and runs it through the sink. It +// reports ok=false (leave the original text) for empty, data:, or unparseable +// references. +func resolveRef(base *url.URL, ref string, sink RefSink) (string, bool) { + ref = strings.TrimSpace(ref) + if ref == "" || strings.HasPrefix(strings.ToLower(ref), "data:") || strings.HasPrefix(ref, "#") { + return "", false + } + u, err := urlx.Normalize(base, ref) + if err != nil { + return "", false + } + return sink(u, urlx.Asset), true +} + +func unquote(s string) string { + s = strings.TrimSpace(s) + if len(s) >= 2 && (s[0] == '"' || s[0] == '\'') && s[len(s)-1] == s[0] { + return s[1 : len(s)-1] + } + return s +} diff --git a/asset/download.go b/asset/download.go new file mode 100644 index 0000000..b8c2112 --- /dev/null +++ b/asset/download.go @@ -0,0 +1,185 @@ +package asset + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// Downloader fetches asset bytes over plain HTTP. It is separate from the Chrome +// pool: assets are public bytes that rarely need a real browser, so a fast HTTP +// client keeps the crawl cheap. Failures are returned to the caller, which logs +// them and moves on — a missing asset degrades a page, it never aborts a clone. +type Downloader struct { + Client *http.Client + UserAgent string + MaxBytes int64 // per-asset cap; 0 = unlimited + Retries int // extra attempts for a transient failure (0 = try once) +} + +// NewDownloader builds a Downloader with a sane client and the given timeout. +func NewDownloader(userAgent string, timeout time.Duration, maxBytes int64) *Downloader { + return &Downloader{ + Client: &http.Client{Timeout: timeout}, + UserAgent: userAgent, + MaxBytes: maxBytes, + // A few sites (and the bot-protection in front of them) reject the first + // request of a burst with a 403 or 429 but serve a retry fine, so give + // transient failures a couple of extra tries before giving up. + Retries: 3, + } +} + +// Result is a downloaded asset. +type Result struct { + Body []byte + ContentType string + IsCSS bool +} + +// ErrTooLarge reports that an asset exceeds the size cap and was skipped without +// being saved. It is deliberately a skip, not a download failure: the caller +// leaves the asset out of the mirror rather than writing a truncated fragment of +// it, so a 500 MB installer or video never bloats the archive with a corrupt +// quarter of itself. +var ErrTooLarge = errors.New("asset over size cap") + +// StatusError reports a non-2xx HTTP response. It carries the code so callers +// can render a clear message ("HTTP 403 Forbidden") and decide whether a retry +// is worthwhile, without the URL baked in (the caller already has it). +type StatusError struct { + Code int +} + +func (e *StatusError) Error() string { + if t := http.StatusText(e.Code); t != "" { + return fmt.Sprintf("HTTP %d %s", e.Code, t) + } + return fmt.Sprintf("HTTP %d", e.Code) +} + +// Get fetches u, sending referer as the Referer header. It reads at most +// MaxBytes and reports whether the body is CSS (so the caller can rewrite it). +// A transient failure (a 403/429/5xx or a network blip) is retried with a short +// backoff up to Retries times. +func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Result, error) { + attempts := d.Retries + 1 + if attempts < 1 { + attempts = 1 + } + var lastErr error + for i := 0; i < attempts; i++ { + if err := ctx.Err(); err != nil { + return nil, err + } + if i > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(backoff(i)): + } + } + res, err := d.try(ctx, u, referer) + if err == nil { + return res, nil + } + lastErr = err + if !transient(err) { + break + } + } + return nil, lastErr +} + +// try performs a single fetch attempt. +func (d *Downloader) try(ctx context.Context, u *url.URL, referer string) (*Result, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + if d.UserAgent != "" { + req.Header.Set("User-Agent", d.UserAgent) + } + if referer != "" { + req.Header.Set("Referer", referer) + } + resp, err := d.Client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, &StatusError{Code: resp.StatusCode} + } + // Skip an over-cap asset instead of truncating it. A Content-Length lets us + // bail before reading a byte; otherwise we read one byte past the cap and, if + // the body really is larger, discard what we have. Either way nothing partial + // reaches disk. + if d.MaxBytes > 0 && resp.ContentLength > d.MaxBytes { + return nil, ErrTooLarge + } + var r io.Reader = resp.Body + if d.MaxBytes > 0 { + // Read at most one byte past the cap so a body with no (or a lying) + // Content-Length cannot stream gigabytes into memory before we notice. + r = io.LimitReader(resp.Body, d.MaxBytes+1) + } + body, err := io.ReadAll(r) + if err != nil { + return nil, err + } + if d.MaxBytes > 0 && int64(len(body)) > d.MaxBytes { + return nil, ErrTooLarge + } + ct := resp.Header.Get("Content-Type") + return &Result{ + Body: body, + ContentType: ct, + IsCSS: isCSS(ct, u), + }, nil +} + +// backoff returns the pause before retry attempt i (1-based): 500ms, 1s, 2s, … +func backoff(i int) time.Duration { + d := 500 * time.Millisecond << (i - 1) + if max := 5 * time.Second; d > max { + d = max + } + return d +} + +// transient reports whether an error is worth retrying. Bot-protection statuses +// (403/429), request-timeout and too-early (408/425), and 5xx server errors are +// transient; other 4xx (404, 401, 410, …) are permanent. A network error is +// retried, but a cancelled or expired context is not. +func transient(err error) bool { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + if errors.Is(err, ErrTooLarge) { + return false + } + var se *StatusError + if errors.As(err, &se) { + switch se.Code { + case http.StatusForbidden, http.StatusRequestTimeout, http.StatusTooEarly, http.StatusTooManyRequests: + return true + } + return se.Code >= 500 + } + return true +} + +// isCSS reports whether a response is a stylesheet, by content-type or by a +// .css path when the server sends no useful type. +func isCSS(contentType string, u *url.URL) bool { + if strings.Contains(strings.ToLower(contentType), "text/css") { + return true + } + return strings.HasSuffix(strings.ToLower(u.Path), ".css") +} diff --git a/asset/download_test.go b/asset/download_test.go new file mode 100644 index 0000000..83fd70d --- /dev/null +++ b/asset/download_test.go @@ -0,0 +1,175 @@ +package asset + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" + "time" +) + +func TestStatusErrorMessage(t *testing.T) { + cases := map[int]string{ + 403: "HTTP 403 Forbidden", + 404: "HTTP 404 Not Found", + 999: "HTTP 999", + } + for code, want := range cases { + if got := (&StatusError{Code: code}).Error(); got != want { + t.Errorf("StatusError{%d} = %q; want %q", code, got, want) + } + } +} + +func TestGetRetriesTransientThenSucceeds(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // 403 on the first try (like bot-protection), then serve the file. + if atomic.AddInt32(&hits, 1) == 1 { + w.WriteHeader(http.StatusForbidden) + return + } + w.Header().Set("Content-Type", "text/css") + _, _ = w.Write([]byte("body{}")) + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 0) + u, _ := url.Parse(srv.URL + "/style.css") + res, err := d.Get(context.Background(), u, "") + if err != nil { + t.Fatalf("Get after retry: %v", err) + } + if !res.IsCSS || string(res.Body) != "body{}" { + t.Errorf("unexpected result: css=%v body=%q", res.IsCSS, res.Body) + } + if hits < 2 { + t.Errorf("expected a retry; server saw %d hits", hits) + } +} + +func TestGetDoesNotRetryPermanent(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 0) + u, _ := url.Parse(srv.URL + "/missing.png") + _, err := d.Get(context.Background(), u, "") + + var se *StatusError + if !errors.As(err, &se) || se.Code != 404 { + t.Fatalf("got %v; want StatusError 404", err) + } + if hits != 1 { + t.Errorf("404 should not be retried; server saw %d hits", hits) + } +} + +func TestGetGivesUpAfterRetries(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + w.WriteHeader(http.StatusTooManyRequests) + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 0) + d.Retries = 2 + u, _ := url.Parse(srv.URL + "/rate.css") + _, err := d.Get(context.Background(), u, "") + + var se *StatusError + if !errors.As(err, &se) || se.Code != 429 { + t.Fatalf("got %v; want StatusError 429", err) + } + if hits != 3 { // 1 try + 2 retries + t.Errorf("expected 3 attempts, server saw %d", hits) + } +} + +func TestGetSkipsOverCapByContentLength(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + body := make([]byte, 4096) // declared via Content-Length + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write(body) + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 1024) // cap below the body + u, _ := url.Parse(srv.URL + "/clip.mp4") + _, err := d.Get(context.Background(), u, "") + if !errors.Is(err, ErrTooLarge) { + t.Fatalf("got %v; want ErrTooLarge", err) + } + if hits != 1 { + t.Errorf("an over-cap asset should not be retried; server saw %d hits", hits) + } +} + +func TestGetSkipsOverCapWithoutContentLength(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // A chunked response carries no Content-Length, so the cap is only known + // after reading past it. + w.Header().Set("Content-Type", "application/octet-stream") + fl, _ := w.(http.Flusher) + chunk := make([]byte, 512) + for i := 0; i < 8; i++ { + _, _ = w.Write(chunk) + if fl != nil { + fl.Flush() + } + } + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 1024) + u, _ := url.Parse(srv.URL + "/stream.bin") + _, err := d.Get(context.Background(), u, "") + if !errors.Is(err, ErrTooLarge) { + t.Fatalf("got %v; want ErrTooLarge", err) + } +} + +func TestGetKeepsUnderCap(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write([]byte("small")) + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 1024) + u, _ := url.Parse(srv.URL + "/logo.png") + res, err := d.Get(context.Background(), u, "") + if err != nil { + t.Fatalf("under-cap asset: %v", err) + } + if string(res.Body) != "small" { + t.Errorf("body = %q; want %q", res.Body, "small") + } +} + +func TestTransientClassification(t *testing.T) { + transientCodes := []int{403, 408, 425, 429, 500, 502, 503} + for _, c := range transientCodes { + if !transient(&StatusError{Code: c}) { + t.Errorf("status %d should be transient", c) + } + } + for _, c := range []int{400, 401, 404, 410} { + if transient(&StatusError{Code: c}) { + t.Errorf("status %d should be permanent", c) + } + } + if transient(context.Canceled) { + t.Error("context.Canceled should not be transient") + } +} diff --git a/asset/html.go b/asset/html.go new file mode 100644 index 0000000..ddbe040 --- /dev/null +++ b/asset/html.go @@ -0,0 +1,161 @@ +package asset + +import ( + "net/url" + "strings" + + "github.com/tamnd/kage/urlx" + "golang.org/x/net/html" + "golang.org/x/net/html/atom" +) + +// assetRels are the individual <link rel> tokens whose href kage downloads as +// an asset. A rel attribute is a space-separated token list, so a single known +// token in it (for example the "stylesheet" in "preload stylesheet") is enough. +var assetRels = map[string]bool{ + "stylesheet": true, "icon": true, + "apple-touch-icon": true, "apple-touch-icon-precomposed": true, + "mask-icon": true, "manifest": true, "preload": true, "prefetch": true, +} + +// linkRelDownloadable reports whether a <link rel> names a resource kage should +// download. It treats rel as the space-separated token list the HTML spec +// defines, so "preload stylesheet", "shortcut icon", or a bare "stylesheet" all +// match on a single recognised token. +func linkRelDownloadable(rel string) bool { + for _, tok := range strings.Fields(strings.ToLower(rel)) { + if assetRels[tok] { + return true + } + } + return false +} + +// RewriteHTML walks the parsed document and rewrites every resource and link +// reference through sink, resolving relative URLs against base. It mutates the +// tree in place; the caller renders it afterwards. References kage cannot handle +// (data:, mailto:, fragment-only, …) are left untouched. +func RewriteHTML(root *html.Node, base *url.URL, sink RefSink) { + var walk func(n *html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode { + rewriteElement(n, base, sink) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(root) +} + +func rewriteElement(n *html.Node, base *url.URL, sink RefSink) { + switch n.DataAtom { + case atom.A, atom.Area: + rewriteAttr(n, "href", base, sink, pageOrAsset) + case atom.Iframe, atom.Frame: + rewriteAttr(n, "src", base, sink, pageOrAsset) + case atom.Link: + if linkRelDownloadable(attrVal(n, "rel")) { + rewriteAttr(n, "href", base, sink, alwaysAsset) + } + case atom.Img: + rewriteAttr(n, "src", base, sink, alwaysAsset) + rewriteSrcset(n, base, sink) + case atom.Source: + rewriteAttr(n, "src", base, sink, alwaysAsset) + rewriteSrcset(n, base, sink) + case atom.Video: + rewriteAttr(n, "src", base, sink, alwaysAsset) + rewriteAttr(n, "poster", base, sink, alwaysAsset) + case atom.Audio, atom.Track, atom.Embed: + rewriteAttr(n, "src", base, sink, alwaysAsset) + case atom.Object: + rewriteAttr(n, "data", base, sink, alwaysAsset) + case atom.Style: + rewriteStyleText(n, base, sink) + } + // Any element may carry an inline style="" with url() references. + rewriteInlineStyle(n, base, sink) +} + +// kindFunc decides whether a normalized URL is a page or an asset. +type kindFunc func(u *url.URL) urlx.Kind + +func alwaysAsset(*url.URL) urlx.Kind { return urlx.Asset } + +func pageOrAsset(u *url.URL) urlx.Kind { + if urlx.LikelyPage(u) { + return urlx.Page + } + return urlx.Asset +} + +func rewriteAttr(n *html.Node, key string, base *url.URL, sink RefSink, kf kindFunc) { + for i := range n.Attr { + if !strings.EqualFold(n.Attr[i].Key, key) { + continue + } + u, err := urlx.Normalize(base, n.Attr[i].Val) + if err != nil { + return + } + n.Attr[i].Val = sink(u, kf(u)) + return + } +} + +// rewriteSrcset rewrites each candidate URL in a srcset attribute, keeping the +// width/density descriptors intact. +func rewriteSrcset(n *html.Node, base *url.URL, sink RefSink) { + for i := range n.Attr { + if !strings.EqualFold(n.Attr[i].Key, "srcset") { + continue + } + n.Attr[i].Val = rewriteSrcsetValue(n.Attr[i].Val, base, sink) + return + } +} + +func rewriteSrcsetValue(val string, base *url.URL, sink RefSink) string { + parts := strings.Split(val, ",") + for i, p := range parts { + fields := strings.Fields(strings.TrimSpace(p)) + if len(fields) == 0 { + continue + } + u, err := urlx.Normalize(base, fields[0]) + if err != nil { + continue + } + fields[0] = sink(u, urlx.Asset) + parts[i] = strings.Join(fields, " ") + } + return strings.Join(parts, ", ") +} + +func rewriteInlineStyle(n *html.Node, base *url.URL, sink RefSink) { + for i := range n.Attr { + if !strings.EqualFold(n.Attr[i].Key, "style") { + continue + } + n.Attr[i].Val = string(RewriteCSS([]byte(n.Attr[i].Val), base, sink)) + return + } +} + +func rewriteStyleText(n *html.Node, base *url.URL, sink RefSink) { + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.TextNode { + c.Data = string(RewriteCSS([]byte(c.Data), base, sink)) + } + } +} + +func attrVal(n *html.Node, key string) string { + for _, a := range n.Attr { + if strings.EqualFold(a.Key, key) { + return a.Val + } + } + return "" +} diff --git a/browser/leakless.go b/browser/leakless.go new file mode 100644 index 0000000..f1f6944 --- /dev/null +++ b/browser/leakless.go @@ -0,0 +1,7 @@ +package browser + +import "runtime" + +func launcherLeakless() bool { + return runtime.GOOS != "windows" +} diff --git a/browser/pool.go b/browser/pool.go new file mode 100644 index 0000000..fc43566 --- /dev/null +++ b/browser/pool.go @@ -0,0 +1,489 @@ +// Package browser drives a real headless Chrome through the DevTools Protocol so +// JavaScript-built pages are captured as they actually render. kage always goes +// through here: navigate, let the page settle, then serialise the final DOM — +// the same markup a human would have seen — which the rest of the pipeline then +// strips of scripts and localises. +package browser + +import ( + "context" + "fmt" + "os" + "runtime" + "strings" + "sync" + "time" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/proto" + "github.com/go-rod/stealth" +) + +// Options configure a Pool. +type Options struct { + Headless bool // run Chrome without a window + Workers int // max concurrent pages + Settle time.Duration // network-idle quiet period after load + RenderTimeout time.Duration // hard cap per page render + Scroll bool // auto-scroll to trigger lazy-loaded media + ChromeBin string // explicit binary; empty = autodetect + ControlURL string // attach to an existing Chrome instead of launching +} + +// DefaultOptions returns the baseline render settings. +func DefaultOptions() Options { + return Options{ + Headless: true, + Workers: 4, + Settle: 1500 * time.Millisecond, + RenderTimeout: 30 * time.Second, + } +} + +// Pool owns one Chrome process shared across a run and bounds the number of +// pages open at once. +type Pool struct { + opts Options + sem chan struct{} + + mu sync.Mutex + browser *rod.Browser + closed bool +} + +// New creates a Pool. Chrome is launched lazily on the first Render. +func New(opts Options) *Pool { + if opts.Workers < 1 { + opts.Workers = 1 + } + return &Pool{opts: opts, sem: make(chan struct{}, opts.Workers)} +} + +// RenderResult is the outcome of rendering one page. +type RenderResult struct { + HTML string // the serialised final DOM + FinalURL string // URL after any client-side redirects + Title string +} + +// ErrNotHTML reports that a URL kage tried to render as a page is not HTML: the +// server returned some other content type (a zip, a CSV, a PDF, a bare image). +// Such a URL reaches the page worker when its link carried no file extension to +// classify it by. The caller reroutes it to the asset downloader, where the +// asset policy decides whether to localise or leave it remote, instead of saving +// an empty or broken page or letting Chrome download it (issue #32). +type ErrNotHTML struct { + URL string + ContentType string +} + +func (e *ErrNotHTML) Error() string { + return fmt.Sprintf("not HTML (%s): %s", e.ContentType, e.URL) +} + +// Render navigates to rawURL, lets it settle, and returns the final rendered +// HTML. It acquires a page slot from the pool and releases it when done. +func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error) { + select { + case p.sem <- struct{}{}: + defer func() { <-p.sem }() + case <-ctx.Done(): + return RenderResult{}, ctx.Err() + } + + b, err := p.getBrowser() + if err != nil { + return RenderResult{}, err + } + + page, err := stealth.Page(b) + if err != nil { + return RenderResult{}, fmt.Errorf("new page: %w", err) + } + defer func() { _ = page.Close() }() + + page = page.Context(ctx).Timeout(p.opts.RenderTimeout) + + // Watch the main document's response so a navigation that turns out to be a + // non-HTML resource (a zip, a CSV, a bare image) is caught and handed back for + // the asset downloader, rather than rendered as a broken page or, with downloads + // denied, left as an aborted navigation (issue #32). The content type arrives in + // the response headers whether Chrome renders the body or aborts it as a denied + // download, so this catches both. + mainContentType := watchMainDocument(page) + + navErr := page.Navigate(rawURL) + // A denied download aborts the navigation, so inspect the captured content type + // before treating a navigation error as a failure. waitFor gives the response + // event a brief moment to be processed; for an HTML page it returns at once. + if ct := waitFor(ctx, mainContentType, 2*time.Second); ct != "" && !isHTML(ct) { + return RenderResult{}, &ErrNotHTML{URL: rawURL, ContentType: ct} + } + if navErr != nil { + return RenderResult{}, fmt.Errorf("navigate %s: %w", rawURL, navErr) + } + if err := page.WaitLoad(); err != nil { + // Chrome's DevTools Protocol may return "Object reference chain is too + // long" when a page's JavaScript builds deeply nested object graphs. + // The page has still loaded its HTML — the error is only about Chrome's + // internal object tracking, not about the document. Log the warning and + // continue rendering rather than failing the entire page (issue #36). + if !isObjRefChainError(err) { + return RenderResult{}, fmt.Errorf("wait load %s: %w", rawURL, err) + } + } + settle(page, p.opts.Settle) + if p.opts.Scroll { + autoScroll(page) + settle(page, p.opts.Settle) + } + + html, err := page.HTML() + if err != nil { + return RenderResult{}, fmt.Errorf("serialise %s: %w", rawURL, err) + } + + res := RenderResult{HTML: html, FinalURL: rawURL} + if info, err := page.Info(); err == nil && info != nil { + res.FinalURL = info.URL + res.Title = info.Title + } + return res, nil +} + +// getBrowser lazily connects to or launches Chrome. +func (p *Pool) getBrowser() (*rod.Browser, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { + return nil, fmt.Errorf("pool is closed") + } + if p.browser != nil { + return p.browser, nil + } + + controlURL := p.opts.ControlURL + if controlURL == "" { + l := launcher.New().Leakless(launcherLeakless()). + Headless(p.opts.Headless). + Set("disable-blink-features", "AutomationControlled"). + Set("disable-gpu", "") + + // Chrome's sandbox is the main line of defense when rendering pages from + // the open web, so kage keeps it on by default (issue #10). It is dropped + // only where it genuinely cannot initialize: inside a container, or when + // running as root, where Chrome otherwise refuses to start. The decision + // is logged so it is never silent. + if off, reason := disableSandbox(); off { + l = l.Set("no-sandbox", "") + warnSandboxDisabled(reason) + } + + // In a container, the default /dev/shm is only 64 MB, too small for + // Chrome's renderer on large pages, so steer it to a temp file instead. + // Outside a container /dev/shm is roomy and faster, so leave it alone. + // + // The "chrome_crashpad_handler: --database is required" abort seen in + // containers (issue #7) is not fixed here: the crash-reporter flags do not + // stop Chrome from spawning the handler. Its real cause is an unwritable + // HOME, which leaves the crash database path empty; the image keeps HOME + // writable instead (see the Dockerfile). + if inContainer() { + l = l.Set("disable-dev-shm-usage", "") + } + + if bin := p.chromeBin(); bin != "" { + l = l.Bin(bin) + } + u, err := l.Launch() + if err != nil { + return nil, fmt.Errorf("launch Chrome: %w", err) + } + controlURL = u + } + + b := rod.New().ControlURL(controlURL) + if err := b.Connect(); err != nil { + return nil, fmt.Errorf("connect Chrome: %w", err) + } + + // kage never wants Chrome to write a file to disk. Every asset is fetched + // through kage's own downloader, which applies the size and media policy, so a + // Chrome-initiated download is only ever an accident: navigating an <a> link + // that turns out to be a binary (a zip, an installer, a CSV) makes Chrome save + // it to the user's Downloads folder, a surprise side effect of a crawl + // (issue #32). Denying downloads browser-wide stops that. The navigation is + // aborted instead, and Render's non-HTML detection reroutes the URL through the + // asset downloader, where the asset policy decides its fate. This is + // best-effort: if the call is unsupported, the non-HTML detection still keeps + // the binary out of the saved mirror. + _ = proto.BrowserSetDownloadBehavior{ + Behavior: proto.BrowserSetDownloadBehaviorBehaviorDeny, + }.Call(b) + + p.browser = b + return b, nil +} + +// Close shuts down the managed Chrome process. +func (p *Pool) Close() error { + p.mu.Lock() + defer p.mu.Unlock() + p.closed = true + if p.browser == nil { + return nil + } + err := p.browser.Close() + p.browser = nil + return err +} + +// LookChrome reports the path of a usable Chrome/Chromium binary and whether one +// was found, checking KAGE_CHROME, CHROME_BIN, rod's own lookup, and the common +// system install locations. Tests use it to skip when no browser is present. +func LookChrome() (string, bool) { + for _, env := range []string{"KAGE_CHROME", "CHROME_BIN"} { + if v := os.Getenv(env); v != "" { + return v, true + } + } + if bin, ok := launcher.LookPath(); ok { + return bin, true + } + for _, c := range systemChromeCandidates() { + if _, err := os.Stat(c); err == nil { + return c, true + } + } + return "", false +} + +// chromeBin returns an explicit Chrome path from options or the environment, or +// "" to let the launcher find/download one. +func (p *Pool) chromeBin() string { + if p.opts.ChromeBin != "" { + return p.opts.ChromeBin + } + for _, env := range []string{"KAGE_CHROME", "CHROME_BIN"} { + if v := os.Getenv(env); v != "" { + return v + } + } + for _, c := range systemChromeCandidates() { + if _, err := os.Stat(c); err == nil { + return c + } + } + return "" +} + +func systemChromeCandidates() []string { + switch runtime.GOOS { + case "darwin": + return []string{ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + } + case "windows": + return []string{ + `C:\Program Files\Google\Chrome\Application\chrome.exe`, + `C:\Program Files (x86)\Google\Chrome\Application\chrome.exe`, + } + default: + return []string{ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + } + } +} + +// disableSandbox decides whether Chrome should launch without its sandbox, with +// a short reason for the log. The secure default is to keep the sandbox on; it +// is dropped only where it cannot run: inside a container, or when running as +// root (Chrome refuses to start a sandbox as root). +func disableSandbox() (off bool, reason string) { + if inContainer() { + return true, "container" + } + if isRoot() { + return true, "root" + } + return false, "" +} + +// warnSandboxDisabled prints why the sandbox was turned off, so dropping a +// security boundary is always visible rather than silent. +func warnSandboxDisabled(reason string) { + switch reason { + case "container": + fmt.Fprintln(os.Stderr, "kage: container detected, Chrome sandbox disabled") + case "root": + fmt.Fprintln(os.Stderr, "kage: running as root, Chrome sandbox disabled (run as a non-root user to keep it on)") + } +} + +// inContainer reports whether kage is running inside a container, where Chrome +// needs container-specific flags. It honors IN_DOCKER (set it in your image) +// and the /.dockerenv marker that Docker writes into every container. +// +// Keeping the sandbox on by default and dropping it only here was prompted by +// Dimitrios Prasakis (issue #10); the IN_DOCKER opt-in was suggested on Hacker +// News (https://news.ycombinator.com/item?id=48534865). Thanks to both. +func inContainer() bool { + if envTrue("IN_DOCKER") { + return true + } + if _, err := os.Stat("/.dockerenv"); err == nil { + return true + } + return false +} + +// isRoot reports whether the process runs as the superuser. On Windows +// os.Geteuid returns -1, so this is false there. +func isRoot() bool { + return os.Geteuid() == 0 +} + +// envTrue reports whether the named environment variable is set to a truthy +// value. +func envTrue(name string) bool { + v, ok := envBool(name) + return ok && v +} + +// envBool parses a boolean-ish environment variable. It returns ok=false when +// the variable is unset or empty. "1", "true", "yes", "on" are true and "0", +// "false", "no", "off" are false (case-insensitive); any other non-empty value +// counts as true, so IN_DOCKER=docker reads as set. +func envBool(name string) (val, ok bool) { + s := strings.TrimSpace(os.Getenv(name)) + if s == "" { + return false, false + } + switch strings.ToLower(s) { + case "1", "true", "yes", "on": + return true, true + case "0", "false", "no", "off": + return false, true + default: + return true, true + } +} + +// watchMainDocument subscribes to network responses and returns an accessor for +// the main document's content type. The first Document-type response is the main +// frame's navigation; later Document responses are sub-frames (iframes), whose +// type kage does not police, so only the first is kept. The accessor is safe to +// call from another goroutine. Any setup error leaves the accessor returning "", +// which the caller reads as "unknown, render normally". +func watchMainDocument(page *rod.Page) func() string { + var ( + mu sync.Mutex + ct string + ) + if err := (proto.NetworkEnable{}).Call(page); err != nil { + return func() string { return "" } + } + wait := page.EachEvent(func(e *proto.NetworkResponseReceived) { + if e.Type != proto.NetworkResourceTypeDocument || e.Response == nil { + return + } + mu.Lock() + if ct == "" { + ct = e.Response.MIMEType + } + mu.Unlock() + }) + // EachEvent's wait blocks until the page context ends, draining events as they + // arrive; run it for the page's lifetime. The deferred page.Close in Render + // cancels the context and unblocks it. + go wait() + return func() string { + mu.Lock() + defer mu.Unlock() + return ct + } +} + +// waitFor polls get until it returns a non-empty value, the deadline passes, or +// the context is cancelled, then returns whatever it last saw. It exists because +// the network response is processed on another goroutine, so the value may not be +// set the instant Navigate returns; an HTML page sets it within a few +// milliseconds, while a never-arriving response simply waits out the deadline. +func waitFor(ctx context.Context, get func() string, deadline time.Duration) string { + const step = 20 * time.Millisecond + for waited := time.Duration(0); waited < deadline; waited += step { + if v := get(); v != "" { + return v + } + select { + case <-ctx.Done(): + return get() + case <-time.After(step): + } + } + return get() +} + +// isHTML reports whether a document content type is one kage renders and saves as +// a page. HTML and XHTML qualify; an empty type is treated as HTML so an unlabelled +// response still renders. Anything else (a zip, a CSV, a PDF, a bare image or +// JSON) is an asset that reached the page worker because its link carried no file +// extension to classify it by. +func isHTML(contentType string) bool { + mt := strings.ToLower(strings.TrimSpace(contentType)) + if i := strings.IndexByte(mt, ';'); i >= 0 { + mt = strings.TrimSpace(mt[:i]) + } + return mt == "" || mt == "text/html" || mt == "application/xhtml+xml" +} + +// isObjRefChainError reports whether err is the Chrome DevTools Protocol error +// "Object reference chain is too long" (code -32000). This surfaces when a +// page's JavaScript builds deeply nested object graphs. The page has still +// loaded — Chrome's internal state tracking hit a limit, not the document +// itself (issue #36). +func isObjRefChainError(err error) bool { + return err != nil && strings.Contains(err.Error(), "Object reference chain is too long") +} + +// settle waits for the network to go quiet for d, recovering from any rod +// panic and capping the wait so a chatty page can never hang the worker. +func settle(page *rod.Page, d time.Duration) { + if d <= 0 { + return + } + defer func() { _ = recover() }() + done := make(chan struct{}) + go func() { + defer func() { _ = recover(); close(done) }() + wait := page.WaitRequestIdle(d, nil, nil, []proto.NetworkResourceType{}) + wait() + }() + select { + case <-done: + case <-time.After(d + 5*time.Second): + } +} + +// autoScroll scrolls to the bottom in steps to trigger lazy-loaded images. +func autoScroll(page *rod.Page) { + defer func() { _ = recover() }() + _, _ = page.Eval(`() => new Promise((resolve) => { + let total = 0; + const step = 800; + const timer = setInterval(() => { + window.scrollBy(0, step); + total += step; + if (total >= document.body.scrollHeight) { + clearInterval(timer); + window.scrollTo(0, 0); + resolve(true); + } + }, 100); + })`) +} diff --git a/browser/pool_test.go b/browser/pool_test.go new file mode 100644 index 0000000..4c14a78 --- /dev/null +++ b/browser/pool_test.go @@ -0,0 +1,210 @@ +package browser + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "runtime" + "strings" + "testing" + "time" +) + +func TestLookChromeReadsEnv(t *testing.T) { + t.Setenv("KAGE_CHROME", "/custom/chrome") + bin, ok := LookChrome() + if !ok || bin != "/custom/chrome" { + t.Fatalf("LookChrome() = %q, %v; want /custom/chrome, true", bin, ok) + } +} + +func TestEnvBool(t *testing.T) { + cases := []struct { + in string + set bool + wantVal bool + wantOk bool + }{ + {"", false, false, false}, + {"1", true, true, true}, + {"true", true, true, true}, + {"TRUE", true, true, true}, + {"yes", true, true, true}, + {"on", true, true, true}, + {"0", true, false, true}, + {"false", true, false, true}, + {"off", true, false, true}, + {"no", true, false, true}, + {"docker", true, true, true}, // any other non-empty value is true + {" true ", true, true, true}, // trimmed + } + for _, c := range cases { + if c.set { + t.Setenv("KAGE_TEST_BOOL", c.in) + } else { + _ = os.Unsetenv("KAGE_TEST_BOOL") + } + val, ok := envBool("KAGE_TEST_BOOL") + if val != c.wantVal || ok != c.wantOk { + t.Errorf("envBool(%q) = (%v, %v); want (%v, %v)", c.in, val, ok, c.wantVal, c.wantOk) + } + } +} + +func TestDisableSandboxDefaultKeepsItOn(t *testing.T) { + // Not in a container and not root, the sandbox stays on. (When the test + // itself runs as root, e.g. some CI containers, "root" is the honest + // reason; accept that rather than asserting a false negative.) + t.Setenv("IN_DOCKER", "") + off, reason := disableSandbox() + if isRoot() || inContainer() { + if !off { + t.Errorf("disableSandbox() = false as root/container; want true") + } + return + } + if off { + t.Errorf("disableSandbox() = true (%q) on a normal host; want sandbox kept on", reason) + } +} + +func TestInContainerHonorsEnv(t *testing.T) { + t.Setenv("IN_DOCKER", "1") + if !inContainer() { + t.Errorf("inContainer() = false with IN_DOCKER=1; want true") + } +} + +func TestDisableSandboxContainer(t *testing.T) { + t.Setenv("IN_DOCKER", "true") + if off, reason := disableSandbox(); !off || reason != "container" { + t.Errorf("in container: got (%v, %q); want (true, container)", off, reason) + } +} + +func TestLauncherLeaklessDisabledOnWindows(t *testing.T) { + got := launcherLeakless() + want := runtime.GOOS != "windows" + if got != want { + t.Errorf("launcherLeakless() = %v on %s; want %v", got, runtime.GOOS, want) + } +} + +func TestRenderCapturesFinalDOM(t *testing.T) { + if testing.Short() { + t.Skip("render test drives Chrome; skipped under -short") + } + if _, ok := LookChrome(); !ok { + t.Skip("no Chrome/Chromium found; skipping render test") + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + // A page whose visible content is built by JavaScript: only a real + // browser render captures the injected node. + _, _ = w.Write([]byte(`<!doctype html><html><body> +<div id="app"></div> +<script>document.getElementById("app").textContent = "rendered-by-js";</script> +</body></html>`)) + })) + defer srv.Close() + + p := New(Options{Headless: true, Workers: 1, Settle: 300 * time.Millisecond, RenderTimeout: 20 * time.Second}) + defer func() { _ = p.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + res, err := p.Render(ctx, srv.URL) + if err != nil { + t.Fatalf("render: %v", err) + } + if !strings.Contains(res.HTML, "rendered-by-js") { + t.Errorf("render did not capture the JS-built DOM:\n%s", res.HTML) + } +} + +func TestIsHTML(t *testing.T) { + cases := []struct { + ct string + want bool + }{ + {"text/html", true}, + {"text/html; charset=utf-8", true}, + {"TEXT/HTML", true}, + {" text/html ", true}, + {"application/xhtml+xml", true}, + {"", true}, // unknown: render rather than misclassify + {"application/zip", false}, + {"text/csv", false}, + {"application/pdf", false}, + {"image/png", false}, + {"application/json", false}, + {"application/octet-stream", false}, + } + for _, c := range cases { + if got := isHTML(c.ct); got != c.want { + t.Errorf("isHTML(%q) = %v, want %v", c.ct, got, c.want) + } + } +} + +func TestRenderRoutesNonHTML(t *testing.T) { + if testing.Short() { + t.Skip("render test drives Chrome; skipped under -short") + } + if _, ok := LookChrome(); !ok { + t.Skip("no Chrome/Chromium found; skipping render test") + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/page": + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(`<!doctype html><html><body><p>a real page</p></body></html>`)) + case "/file.zip", "/download": + // A binary served with no useful extension on the path, the shape that + // makes Chrome download to ~/Downloads when navigated to (issue #32). + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write([]byte("PK\x03\x04 not really a zip")) + case "/data": + w.Header().Set("Content-Type", "text/csv") + _, _ = w.Write([]byte("a,b\n1,2\n")) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + p := New(Options{Headless: true, Workers: 1, Settle: 300 * time.Millisecond, RenderTimeout: 20 * time.Second}) + defer func() { _ = p.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + // A real HTML page renders as before. + if res, err := p.Render(ctx, srv.URL+"/page"); err != nil { + t.Errorf("render HTML page: %v", err) + } else if !strings.Contains(res.HTML, "a real page") { + t.Errorf("HTML page did not render:\n%s", res.HTML) + } + + // Non-HTML navigation targets come back as *ErrNotHTML so the caller can route + // them to the asset downloader instead of saving a broken page or downloading. + for _, tc := range []struct{ path, wantCT string }{ + {"/download", "application/zip"}, + {"/data", "text/csv"}, + } { + _, err := p.Render(ctx, srv.URL+tc.path) + var notHTML *ErrNotHTML + if !errors.As(err, ¬HTML) { + t.Errorf("Render(%s) error = %v, want *ErrNotHTML", tc.path, err) + continue + } + if !strings.Contains(notHTML.ContentType, tc.wantCT) { + t.Errorf("Render(%s) content type = %q, want %q", tc.path, notHTML.ContentType, tc.wantCT) + } + } +} diff --git a/cli/clone.go b/cli/clone.go new file mode 100644 index 0000000..a3b402a --- /dev/null +++ b/cli/clone.go @@ -0,0 +1,260 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/tamnd/kage/clone" + "github.com/tamnd/kage/urlx" +) + +// cloneFlags holds the parsed flag values for one invocation. +type cloneFlags struct { + out string + reserved string + workers int + assetWorkers int + browserPages int + maxPages int + maxDepth int + traversal string + maxAssetMB int64 + keepMedia bool + skipExt []string + allAssetHosts bool + timeout time.Duration + settle time.Duration + renderTO time.Duration + scroll bool + userAgent string + subdomains bool + scopePrefix string + exclude []string + noRobots bool + crawlDelay time.Duration + noSitemap bool + headful bool + keepNoscript bool + mobileReadable bool + chromeBin string + controlURL string + noResume bool + refresh bool + force bool + quiet bool +} + +func newCloneCmd() *cobra.Command { + f := &cloneFlags{} + cmd := &cobra.Command{ + Use: "clone <url>", + Short: "Clone a site into a self-contained offline folder", + Long: "clone fetches the seed URL, follows in-scope links, and writes a browsable\n" + + "mirror to <out>/<host>/. Every page is rendered in Chrome and stripped of\n" + + "scripts; CSS, images, and fonts are downloaded and rewritten to local paths.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runClone(cmd.Context(), args[0], f) + }, + } + fs := cmd.Flags() + fs.StringVarP(&f.out, "out", "o", clone.DefaultOutDir(), "output root; the mirror lands in <out>/<host>/") + fs.StringVar(&f.reserved, "reserved", urlx.DefaultReserved, "reserved dir for assets and state") + fs.IntVar(&f.workers, "workers", 4, "concurrent page render workers") + fs.IntVar(&f.assetWorkers, "asset-workers", 8, "concurrent asset download workers") + fs.IntVar(&f.browserPages, "browser-pages", 4, "Chrome page-pool size") + fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "stop after N pages (0 = unlimited)") + fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)") + fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order: bfs or dfs") + fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB (left on the live web)") + fs.BoolVar(&f.keepMedia, "keep-media", false, "download bulk media, installers, and PDFs instead of leaving them remote") + fs.StringSliceVar(&f.skipExt, "skip-ext", nil, "extra asset extensions to leave remote, e.g. .svg (repeatable)") + fs.BoolVar(&f.allAssetHosts, "all-asset-hosts", false, "localize assets from any host, not just the seed's domain") + fs.DurationVar(&f.timeout, "timeout", 30*time.Second, "per-request timeout") + fs.DurationVar(&f.settle, "settle", 1500*time.Millisecond, "network-idle quiet period before snapshot") + fs.DurationVar(&f.renderTO, "render-timeout", 30*time.Second, "hard cap per page render") + fs.BoolVar(&f.scroll, "scroll", false, "auto-scroll each page to trigger lazy loading") + fs.StringVar(&f.userAgent, "user-agent", clone.DefaultUserAgent, "User-Agent for asset and robots fetches") + fs.BoolVar(&f.subdomains, "subdomains", false, "treat subdomains of the seed host as in scope") + fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix") + fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)") + fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)") + fs.DurationVar(&f.crawlDelay, "crawl-delay", 0, "override robots.txt Crawl-delay between page starts (0 = use robots.txt)") + fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml") + fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)") + fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap <noscript> content instead of dropping it") + fs.BoolVar(&f.mobileReadable, "mobile", false, "inject viewport and CSS overrides so legacy sites read comfortably on a phone") + fs.StringVar(&f.chromeBin, "chrome", "", "path to the Chrome/Chromium binary") + fs.StringVar(&f.controlURL, "control-url", "", "attach to an existing Chrome DevTools endpoint") + fs.BoolVar(&f.noResume, "no-resume", false, "do not reuse or write resume state") + fs.BoolVar(&f.refresh, "refresh", false, "re-render every page in place to pull in changed content") + fs.BoolVarP(&f.force, "force", "f", false, "delete any existing mirror for the host first") + fs.BoolVarP(&f.quiet, "quiet", "q", false, "suppress per-page progress lines") + return cmd +} + +func runClone(ctx context.Context, arg string, f *cloneFlags) error { + seed, err := urlx.ParseSeed(arg) + if err != nil { + return fmt.Errorf("invalid url %q: %w", arg, err) + } + if f.crawlDelay < 0 { + return fmt.Errorf("--crawl-delay must be >= 0") + } + + cfg := clone.DefaultConfig() + cfg.OutDir = f.out + cfg.Reserved = f.reserved + cfg.Workers = f.workers + cfg.AssetWorkers = f.assetWorkers + cfg.BrowserPages = f.browserPages + cfg.MaxPages = f.maxPages + cfg.MaxDepth = f.maxDepth + cfg.Traversal = f.traversal + cfg.MaxAssetBytes = f.maxAssetMB << 20 + cfg.AssetSameDomain = !f.allAssetHosts + if f.keepMedia { + cfg.SkipAssetExts = map[string]bool{} + } + for _, e := range f.skipExt { + e = strings.ToLower(strings.TrimSpace(e)) + if e == "" { + continue + } + if !strings.HasPrefix(e, ".") { + e = "." + e + } + if cfg.SkipAssetExts == nil { + cfg.SkipAssetExts = map[string]bool{} + } + cfg.SkipAssetExts[e] = true + } + cfg.Timeout = f.timeout + cfg.Settle = f.settle + cfg.RenderTimeout = f.renderTO + cfg.Scroll = f.scroll + cfg.UserAgent = f.userAgent + cfg.IncludeSubdomains = f.subdomains + cfg.ScopePrefix = f.scopePrefix + cfg.ExcludePaths = f.exclude + cfg.RespectRobots = !f.noRobots + cfg.CrawlDelay = f.crawlDelay + cfg.FollowSitemap = !f.noSitemap + cfg.Headless = !f.headful + cfg.KeepNoscript = f.keepNoscript + cfg.MobileReadable = f.mobileReadable + cfg.ChromeBin = f.chromeBin + cfg.ControlURL = f.controlURL + cfg.Resume = !f.noResume + cfg.Persist = !f.noResume + cfg.Refresh = f.refresh + cfg.Force = f.force + + logf := func(format string, args ...any) { + if !f.quiet { + fmt.Fprintln(os.Stderr, styleDim.Render(fmt.Sprintf(format, args...))) + } + } + + fmt.Fprintln(os.Stderr, styleTitle.Render("kage")+" cloning "+styleAccent.Render(seed.String())) + + c := clone.New(seed, cfg, logf) + + // Live progress ticker on a second line, refreshed every second. + tickCtx, stopTicker := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + defer close(done) + if f.quiet { + return + } + t := time.NewTicker(time.Second) + defer t.Stop() + for { + select { + case <-tickCtx.Done(): + return + case <-t.C: + p := c.Snapshot() + fmt.Fprintf(os.Stderr, "\r%s", progressLine(p)) + } + } + }() + + res, runErr := c.Run(ctx) + stopTicker() + <-done + if !f.quiet { + fmt.Fprint(os.Stderr, "\r\033[K") + } + + printSummary(res) + + if runErr != nil && !errors.Is(runErr, context.Canceled) { + return runErr + } + if errors.Is(runErr, context.Canceled) { + fmt.Fprintln(os.Stderr, styleWarn.Render("interrupted; resume state saved (rerun to continue)")) + } + return nil +} + +// progressLine renders the single-line live counter. "pages" is the count of +// real pages (distinct paths); when a faceted site spawns query-string variants +// they are shown separately so the page number stays easy to read. +func progressLine(p clone.Progress) string { + if variants := p.Pages - p.PagePaths; variants > 0 { + return styleDim.Render(fmt.Sprintf("pages %d variants %d assets %d errors %d skipped %d", + p.PagePaths, variants, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped)) + } + return styleDim.Render(fmt.Sprintf("pages %d assets %d errors %d skipped %d", + p.PagePaths, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped)) +} + +// printSummary prints the final tally and where the mirror landed. +func printSummary(res clone.Result) { + fmt.Fprintln(os.Stderr, styleOK.Render("done")+" "+styleTitle.Render(res.OutDir)) + fmt.Fprintf(os.Stderr, " %s %d %s %d\n", + styleAccent.Render("pages"), res.PagePaths, + styleAccent.Render("assets"), res.Assets) + if variants := res.Pages - res.PagePaths; variants > 0 { + fmt.Fprintf(os.Stderr, " %s %d\n", styleDim.Render("query variants"), variants) + } + if res.PagesLinked > 0 { + fmt.Fprintf(os.Stderr, " %s %d\n", styleDim.Render("deduped (linked)"), res.PagesLinked) + } + if res.AssetSkipped > 0 { + fmt.Fprintf(os.Stderr, " %s %d\n", styleDim.Render("assets over cap (left remote)"), res.AssetSkipped) + } + if res.PageErrors+res.AssetErrors > 0 { + fmt.Fprintf(os.Stderr, " %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors) + printFailures(res) + } + if res.Skipped > 0 { + fmt.Fprintf(os.Stderr, " %s %d\n", styleWarn.Render("skipped"), res.Skipped) + } + fmt.Fprintf(os.Stderr, " open %s\n", styleAccent.Render("kage serve "+res.OutDir)) +} + +// printFailures lists what went wrong, grouped reason and URL, so the error +// count is actionable instead of opaque. The list is capped during the crawl; +// when it overflows, say how many more there were. +func printFailures(res clone.Result) { + total := res.PageErrors + res.AssetErrors + for _, f := range res.Failures { + line := fmt.Sprintf(" %s %s", styleErr.Render(f.Reason), f.URL) + fmt.Fprintln(os.Stderr, line) + if f.Referer != "" { + fmt.Fprintln(os.Stderr, styleDim.Render(" referenced by "+f.Referer)) + } + } + if more := total - int64(len(res.Failures)); more > 0 { + fmt.Fprintln(os.Stderr, styleDim.Render(fmt.Sprintf(" ... and %d more", more))) + } +} diff --git a/cli/open.go b/cli/open.go new file mode 100644 index 0000000..6feac08 --- /dev/null +++ b/cli/open.go @@ -0,0 +1,66 @@ +package cli + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + + "github.com/spf13/cobra" + + "github.com/tamnd/kage/pack" + "github.com/tamnd/kage/viewer" + "github.com/tamnd/kage/zim" +) + +func newOpenCmd() *cobra.Command { + var addr string + var openBrowser bool + cmd := &cobra.Command{ + Use: "open <file.zim>", + Short: "Serve a ZIM archive in your browser for offline reading", + Long: "open serves a packed ZIM file over a local HTTP server so you can browse the\n" + + "site exactly as it was cloned. It is the read side of kage pack --format zim.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runOpen(cmd.Context(), args[0], addr, openBrowser) + }, + } + cmd.Flags().StringVarP(&addr, "addr", "a", "127.0.0.1:8800", "address to listen on") + cmd.Flags().BoolVar(&openBrowser, "open", true, "open the default browser") + return cmd +} + +func runOpen(ctx context.Context, path, addr string, openBrowser bool) error { + r, err := zim.Open(path) + if err != nil { + return fmt.Errorf("cannot open %q: %w", path, err) + } + defer func() { _ = r.Close() }() + + ln, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("cannot listen on %s: %w", addr, err) + } + url := "http://" + ln.Addr().String() + + fmt.Fprintln(os.Stderr, styleTitle.Render("kage open")+" "+styleDim.Render(path)) + fmt.Fprintln(os.Stderr, " open "+styleAccent.Render(url)) + if viewer.Native { + fmt.Fprintln(os.Stderr, styleDim.Render(" close the window to stop")) + } else { + fmt.Fprintln(os.Stderr, styleDim.Render(" press Ctrl-C to stop")) + } + + srv := &http.Server{Handler: pack.Handler(r)} + srvErr := make(chan error, 1) + go func() { srvErr <- srv.Serve(ln) }() + + _ = viewer.Show(ctx, viewer.Options{Title: archiveTitle(r), URL: url, Browser: openBrowser}) + _ = srv.Close() + if err := <-srvErr; err != nil && err != http.ErrServerClosed { + return err + } + return nil +} diff --git a/cli/pack.go b/cli/pack.go new file mode 100644 index 0000000..a7ae674 --- /dev/null +++ b/cli/pack.go @@ -0,0 +1,435 @@ +package cli + +import ( + "fmt" + "image" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/tamnd/kage/clone" + "github.com/tamnd/kage/pack" +) + +// packFlags holds the parsed flags for one invocation of kage pack. +type packFlags struct { + format string + out string + base string + app bool + icon string + noCompress bool + title string + description string + language string + date string + incremental bool +} + +// cacheSuffix names the cluster-cache sidecar kage writes next to a packed +// artifact when --incremental is set. +const cacheSuffix = ".kagecache" + +func newPackCmd() *cobra.Command { + f := &packFlags{} + cmd := &cobra.Command{ + Use: "pack <mirror-dir>", + Short: "Pack a cloned mirror into a ZIM file or a self-contained viewer", + Long: "pack turns a cloned folder into one distributable file. With --format zim\n" + + "it writes an open ZIM archive (the format Kiwix uses) that kage open or any\n" + + "ZIM reader can browse. With --format binary it appends that archive to a copy\n" + + "of kage, producing a single executable that serves the site offline when run.\n" + + "Add --app to wrap that executable in a double-click desktop app (a .app bundle\n" + + "on macOS, an AppImage-style .AppDir on Linux) with the site's favicon as the icon.\n" + + "Add --incremental to keep a cache sidecar so re-packing a mirror only compresses\n" + + "the clusters that changed, not the whole archive.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runPack(args[0], f) + }, + } + fs := cmd.Flags() + fs.StringVar(&f.format, "format", "zim", "output format: zim or binary") + fs.StringVarP(&f.out, "out", "o", "", "output path (default per format)") + fs.StringVar(&f.base, "base", "", "base kage binary for the viewer (default this kage)") + fs.BoolVar(&f.app, "app", false, "wrap the viewer in a double-click desktop app (.app on macOS, .AppImage/.AppDir on Linux)") + fs.StringVar(&f.icon, "icon", "", "icon file for --app (default the site's favicon)") + fs.BoolVar(&f.noCompress, "no-compress", false, "store every cluster raw, no zstd") + fs.BoolVar(&f.incremental, "incremental", false, "reuse compression from a cache sidecar so re-packing a mirror only compresses what changed") + fs.StringVar(&f.title, "title", "", "archive title (default the main page's <title>)") + fs.StringVar(&f.description, "description", "", "archive description") + fs.StringVar(&f.language, "language", "eng", "archive language code") + fs.StringVar(&f.date, "date", time.Now().UTC().Format("2006-01-02"), "archive date (YYYY-MM-DD)") + return cmd +} + +func runPack(mirrorArg string, f *packFlags) error { + dir := resolveMirror(mirrorArg) + zopts := pack.ZIMOptions{ + Out: f.out, + NoCompress: f.noCompress, + Title: f.title, + Description: f.description, + Language: f.language, + Date: f.date, + Version: Version, + } + + // --app wraps the packed viewer in a desktop bundle. It builds on the binary + // format, so it owns the flow rather than being one more --format value. + if f.app { + return runPackApp(dir, f, zopts) + } + + switch f.format { + case "zim": + out := f.out + if out == "" { + out = filepath.Base(dir) + ".zim" + } + zopts.Out = out + var st pack.PackStats + if f.incremental { + zopts.CachePath = out + cacheSuffix + zopts.Stats = &st + } + outPath, size, err := pack.BuildZIM(dir, zopts) + if err != nil { + return err + } + printPackResult(outPath, size) + printCacheLine(f.incremental, st) + fmt.Fprintf(os.Stderr, " open %s\n", styleAccent.Render("kage open "+outPath)) + return nil + + case "binary": + target := resolveTargetOS(f.base) + out := f.out + if out == "" { + out = defaultBinaryName(dir) + } + // A Windows viewer must end in .exe to run, whether the name came from + // --out or the default, so make sure it does. + if target == "windows" && !strings.HasSuffix(strings.ToLower(out), ".exe") { + out += ".exe" + } + var st pack.PackStats + if f.incremental { + zopts.CachePath = out + cacheSuffix + zopts.Stats = &st + } + zbytes, err := pack.BuildZIMBytes(dir, zopts) + if err != nil { + return err + } + path, size, err := pack.BuildBinary(zbytes, pack.BinaryOptions{Out: out, Base: f.base}) + if err != nil { + return err + } + printPackResult(path, size) + printCacheLine(f.incremental, st) + printRunHint(path, target) + return nil + + default: + return fmt.Errorf("unknown --format %q (want zim or binary)", f.format) + } +} + +// runPackApp builds a double-clickable desktop app around the packed viewer, +// shaped for whichever OS the base targets: a .app bundle on macOS, an +// AppImage-style .AppDir on Linux. Windows needs no bundle (the .exe is the +// app), so it is redirected to --format binary with a GUI base. +func runPackApp(dir string, f *packFlags, zopts pack.ZIMOptions) error { + target := resolveTargetOS(f.base) + switch target { + case "windows": + return fmt.Errorf("a Windows app is just the .exe, with no bundle to build: use --format binary and a GUI base (kage built with -ldflags -H=windowsgui)") + case "": + if f.base != "" { + return fmt.Errorf("--app could not tell which OS %q is for; pass a macOS or Linux kage as --base", f.base) + } + // No base and an unknown runtime: fall through with the host's GOOS. + target = runtime.GOOS + } + + prog := defaultBinaryName(dir) + name := f.title + if name == "" { + name = prog + } + icon, iconSrc, err := resolveIcon(dir, f.icon) + if err != nil { + return err + } + var st pack.PackStats + if f.incremental { + zopts.CachePath = prog + cacheSuffix + zopts.Stats = &st + } + zbytes, err := pack.BuildZIMBytes(dir, zopts) + if err != nil { + return err + } + printCacheLine(f.incremental, st) + + switch target { + case "darwin": + return packMacApp(zbytes, dir, f, prog, name, icon, iconSrc) + case "linux": + return packLinuxApp(zbytes, dir, f, prog, name, icon, iconSrc) + default: + return fmt.Errorf("--app supports macOS and Linux bases; %s is not one of them", osLabel(target)) + } +} + +// packMacApp writes the .app bundle and prints how to launch it. +func packMacApp(zbytes []byte, dir string, f *packFlags, prog, name string, icon image.Image, iconSrc string) error { + out := f.out + if out == "" { + out = prog + ".app" + } else if !strings.HasSuffix(strings.ToLower(out), ".app") { + out += ".app" + } + path, size, err := pack.BuildApp(zbytes, pack.AppOptions{ + Out: out, + Base: f.base, + Name: name, + ExecName: prog, + Identifier: bundleID(prog), + Version: appVersion(), + Icon: icon, + }) + if err != nil { + return err + } + printPackResult(path, size) + printIconLine(iconSrc) + fmt.Fprintf(os.Stderr, " double-click %s to open the site offline\n", styleAccent.Render(filepath.Base(path))) + if f.base == "" { + fmt.Fprintln(os.Stderr, styleDim.Render(" (built around this kage; pass --base a webview build to open a native window instead of the browser)")) + } + fmt.Fprintln(os.Stderr, styleDim.Render(" (macOS may quarantine it: xattr -dr com.apple.quarantine "+path+")")) + return nil +} + +// packLinuxApp writes the .AppDir and, when appimagetool is installed, folds it +// into a single double-clickable .AppImage. +func packLinuxApp(zbytes []byte, dir string, f *packFlags, prog, name string, icon image.Image, iconSrc string) error { + out := f.out + if out == "" { + out = prog + ".AppDir" + } else if !strings.HasSuffix(out, ".AppDir") { + out += ".AppDir" + } + path, size, hasIcon, err := pack.BuildAppDir(zbytes, pack.LinuxAppOptions{ + Out: out, + Base: f.base, + Name: name, + ExecName: prog, + Comment: f.description, + Version: appVersion(), + Icon: icon, + }) + if err != nil { + return err + } + printPackResult(path, size) + printIconLine(iconSrc) + + // appimagetool turns the directory into one portable file. It needs an icon, + // so only attempt it when the mirror gave us one. + if hasIcon { + if img, ok := tryAppImage(path, prog); ok { + fmt.Fprintf(os.Stderr, " built %s\n", styleTitle.Render(img)) + fmt.Fprintf(os.Stderr, " double-click %s to open the site offline\n", styleAccent.Render(filepath.Base(img))) + return nil + } + } + fmt.Fprintf(os.Stderr, " run %s to open the site offline\n", styleAccent.Render("./"+filepath.Join(filepath.Base(path), "AppRun"))) + fmt.Fprintln(os.Stderr, styleDim.Render(" (install appimagetool to fold this .AppDir into one double-clickable .AppImage)")) + return nil +} + +// tryAppImage runs appimagetool over the AppDir if it is installed, returning +// the .AppImage path on success. A missing tool or a build failure is not fatal: +// the caller falls back to the AppDir. +func tryAppImage(appDir, prog string) (string, bool) { + tool, err := exec.LookPath("appimagetool") + if err != nil { + return "", false + } + out := prog + ".AppImage" + cmd := exec.Command(tool, appDir, out) + // appimagetool reads the target arch from the AppRun ELF; suppress its noisy + // progress so kage's own output stays clean, but surface a real failure. + if err := cmd.Run(); err != nil { + return "", false + } + if _, err := os.Stat(out); err != nil { + return "", false + } + return out, true +} + +func printIconLine(iconSrc string) { + if iconSrc != "" { + fmt.Fprintf(os.Stderr, " icon %s\n", styleDim.Render(iconSrc)) + } +} + +// resolveIcon picks the bundle icon: an explicit --icon path if given (an error +// there is fatal, since the user asked for that file), otherwise the site's +// favicon discovered in the mirror. A mirror with no usable icon is fine; the +// bundle just ships without a custom one. +func resolveIcon(dir, iconFlag string) (img image.Image, src string, err error) { + if iconFlag != "" { + img, err = pack.DecodeIcon(iconFlag) + if err != nil { + return nil, "", err + } + return img, iconFlag, nil + } + if img, src, ok := pack.FindIcon(dir); ok { + return img, src, nil + } + return nil, "", nil +} + +// bundleID builds a reverse-DNS CFBundleIdentifier from the program name, +// keeping only characters Apple allows in an identifier. +func bundleID(prog string) string { + var b strings.Builder + for _, r := range strings.ToLower(prog) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-': + b.WriteRune(r) + default: + b.WriteRune('-') + } + } + id := strings.Trim(b.String(), "-") + if id == "" { + id = "app" + } + return "com.kage." + id +} + +// appVersion uses kage's own version for the bundle, falling back to 1.0 for a +// dev build whose version is the default "dev". +func appVersion() string { + if Version == "" || Version == "dev" { + return "1.0" + } + return strings.TrimPrefix(Version, "v") +} + +// resolveMirror accepts either a path to a mirror dir or a bare host. A bare +// host that is not a directory in the working dir is resolved against the +// default out dir, so "kage pack paulgraham.com" works right after a clone. +func resolveMirror(arg string) string { + if info, err := os.Stat(arg); err == nil && info.IsDir() { + return arg + } + candidate := filepath.Join(clone.DefaultOutDir(), arg) + if info, err := os.Stat(candidate); err == nil && info.IsDir() { + return candidate + } + return arg +} + +// defaultBinaryName derives a clean program name from the mirror's host by +// stripping a trailing dot-suffix (paulgraham.com -> paulgraham). The caller +// appends .exe for Windows targets. +func defaultBinaryName(dir string) string { + host := filepath.Base(dir) + if i := strings.IndexByte(host, '.'); i > 0 { + return host[:i] + } + return host +} + +// resolveTargetOS reports which OS the packed viewer will run on. With no +// --base it is this kage's OS; with one, we sniff the base's executable header +// so detection does not hinge on the file being named ".exe". If the header is +// unrecognised we fall back to that name heuristic. +func resolveTargetOS(base string) string { + if base == "" { + return runtime.GOOS + } + if os := pack.SniffOS(base); os != "" { + return os + } + if strings.HasSuffix(strings.ToLower(base), ".exe") { + return "windows" + } + return "" +} + +func printPackResult(path string, size int64) { + fmt.Fprintln(os.Stderr, styleOK.Render("packed")+" "+styleTitle.Render(path)) + fmt.Fprintf(os.Stderr, " %s %s\n", styleAccent.Render("size"), humanBytes(size)) +} + +// printCacheLine reports how much compression the incremental cache reused. On +// the first pack everything is compressed fresh; on a re-pack after a small +// change most clusters are reused and only the rest are compressed. +func printCacheLine(incremental bool, st pack.PackStats) { + if !incremental { + return + } + total := st.ClustersReused + st.ClustersCompressed + fmt.Fprintf(os.Stderr, " %s %d reused, %d compressed of %d clusters\n", + styleDim.Render("cache"), st.ClustersReused, st.ClustersCompressed, total) +} + +func printRunHint(path, target string) { + rel := path + if !strings.ContainsAny(path, "/\\") { + rel = "./" + path + } + // A viewer built for another OS cannot run here, so say where it goes + // instead of printing a run command that would not work. + if target != "" && target != runtime.GOOS { + fmt.Fprintf(os.Stderr, " this is a %s viewer; copy %s to that machine to run it\n", + osLabel(target), styleAccent.Render(filepath.Base(path))) + return + } + fmt.Fprintf(os.Stderr, " run %s to view the site offline\n", styleAccent.Render(rel)) + if target == "darwin" { + fmt.Fprintln(os.Stderr, styleDim.Render(" (macOS may quarantine it: xattr -d com.apple.quarantine "+rel+")")) + } +} + +// osLabel turns a GOOS value into a friendly name for the run hint. +func osLabel(goos string) string { + switch goos { + case "windows": + return "Windows" + case "darwin": + return "macOS" + case "linux": + return "Linux" + default: + return goos + } +} + +// humanBytes renders a byte count in B, KiB, MiB, or GiB. +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for x := n / unit; x >= unit; x /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} diff --git a/cli/parquet.go b/cli/parquet.go new file mode 100644 index 0000000..8194078 --- /dev/null +++ b/cli/parquet.go @@ -0,0 +1,94 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + + "github.com/tamnd/kage/dataset" +) + +// newParquetCmd groups the two columnar conversions: a ZIM archive out to a +// Parquet table, and a Parquet table back to a ZIM archive. The table is a flat +// one-row-per-entry shape ready to publish as a dataset, and the round trip is +// lossless. +func newParquetCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "parquet", + Short: "Convert a ZIM archive to a Parquet dataset and back", + Long: "parquet converts a packed ZIM archive into a columnar Parquet table, one row\n" + + "per entry with clear columns (url, mime, title, content, extracted text), and\n" + + "converts such a table back into a ZIM. The table is the shape a dataset host\n" + + "like Hugging Face expects, and the conversion is lossless: a ZIM round-tripped\n" + + "through Parquet reproduces every entry, its metadata, and the main page.", + } + cmd.AddCommand(newParquetExportCmd()) + cmd.AddCommand(newParquetImportCmd()) + return cmd +} + +func newParquetExportCmd() *cobra.Command { + var out string + cmd := &cobra.Command{ + Use: "export <file.zim>", + Short: "Write a Parquet table from a ZIM archive", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + in := args[0] + if out == "" { + out = strings.TrimSuffix(in, filepath.Ext(in)) + ".parquet" + } + st, err := dataset.ZIMToParquet(in, out, Version) + if err != nil { + return err + } + printDatasetResult("exported", out) + printDatasetStats(st, out) + return nil + }, + } + cmd.Flags().StringVarP(&out, "out", "o", "", "output path (default <name>.parquet)") + return cmd +} + +func newParquetImportCmd() *cobra.Command { + var out string + cmd := &cobra.Command{ + Use: "import <file.parquet>", + Short: "Rebuild a ZIM archive from a Parquet table", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + in := args[0] + if out == "" { + out = strings.TrimSuffix(in, filepath.Ext(in)) + ".zim" + } + st, err := dataset.ParquetToZIM(in, out, Version) + if err != nil { + return err + } + printDatasetResult("imported", out) + printDatasetStats(st, out) + fmt.Fprintf(os.Stderr, " open %s\n", styleAccent.Render("kage open "+out)) + return nil + }, + } + cmd.Flags().StringVarP(&out, "out", "o", "", "output path (default <name>.zim)") + return cmd +} + +func printDatasetResult(verb, path string) { + fmt.Fprintln(os.Stderr, styleOK.Render(verb)+" "+styleTitle.Render(path)) +} + +func printDatasetStats(st dataset.Stats, path string) { + fmt.Fprintf(os.Stderr, " %s %d %s %d\n", + styleAccent.Render("rows"), st.Rows, + styleAccent.Render("redirects"), st.Redirects) + fmt.Fprintf(os.Stderr, " %s %s content\n", styleDim.Render("content"), humanBytes(st.ContentBytes)) + if fi, err := os.Stat(path); err == nil { + fmt.Fprintf(os.Stderr, " %s %s\n", styleAccent.Render("size"), humanBytes(fi.Size())) + } +} diff --git a/cli/root.go b/cli/root.go new file mode 100644 index 0000000..6fe93ac --- /dev/null +++ b/cli/root.go @@ -0,0 +1,109 @@ +// Package cli wires kage's command surface: the cobra tree, the global flags, +// and the fang-rendered help and errors. The actual work lives in the clone, +// browser, sanitize, asset, and urlx packages; this layer only parses flags and +// prints progress. +package cli + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "os" + + "github.com/charmbracelet/fang" + "github.com/spf13/cobra" + + "github.com/tamnd/kage/pack" + "github.com/tamnd/kage/viewer" + "github.com/tamnd/kage/zim" +) + +// Execute builds the root command and runs it through fang. main passes the +// signal-aware context so Ctrl-C cancels the in-flight clone and flushes resume +// state. It returns the process exit code. +func Execute(ctx context.Context) int { + // A kage binary with a ZIM appended runs as an offline viewer for that site, + // ignoring its arguments. A normal build has no trailer and falls through. + if ra, size, ok := pack.Embedded(); ok { + return runEmbeddedViewer(ctx, ra, size) + } + + root := newRoot() + opts := []fang.Option{ + fang.WithVersion(Version), + } + if err := fang.Execute(ctx, root, opts...); err != nil { + return 1 + } + return 0 +} + +// newRoot assembles the command tree. +func newRoot() *cobra.Command { + root := &cobra.Command{ + Use: "kage", + Short: "Clone any website for offline viewing, with the JavaScript stripped out", + Long: "kage (影, \"shadow\") renders each page in headless Chrome, snapshots the\n" + + "final DOM, removes every script and event handler, and localises the CSS,\n" + + "images, and fonts so the saved copy looks like the live site but runs no\n" + + "code. The result is a plain folder you can open straight from disk.", + Version: fmt.Sprintf("%s (commit %s, built %s)", Version, Commit, Date), + SilenceUsage: true, + SilenceErrors: true, + } + root.AddCommand(newCloneCmd()) + root.AddCommand(newServeCmd()) + root.AddCommand(newPackCmd()) + root.AddCommand(newOpenCmd()) + root.AddCommand(newParquetCmd()) + return root +} + +// runEmbeddedViewer serves the ZIM appended to this executable on an ephemeral +// local port and shows it: a native window in the webview build, the system +// browser otherwise. It runs until the viewer closes or the context is +// cancelled (Ctrl-C) and ignores all command-line arguments, because a packed +// binary is the site, not the kage CLI. +func runEmbeddedViewer(ctx context.Context, ra io.ReaderAt, size int64) int { + r, err := zim.NewReader(ra, size) + if err != nil { + fmt.Fprintln(os.Stderr, "kage: corrupt embedded archive:", err) + return 1 + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + fmt.Fprintln(os.Stderr, "kage: cannot start viewer:", err) + return 1 + } + url := "http://" + ln.Addr().String() + if viewer.Native { + fmt.Fprintln(os.Stderr, "opening offline site (close the window to stop)") + } else { + fmt.Fprintln(os.Stderr, "serving offline site at "+url+" (Ctrl-C to stop)") + } + + srv := &http.Server{Handler: pack.Handler(r)} + srvErr := make(chan error, 1) + go func() { srvErr <- srv.Serve(ln) }() + + // Show blocks until the window closes (native) or ctx is cancelled (browser); + // either way, tear the server down afterwards. + _ = viewer.Show(ctx, viewer.Options{Title: archiveTitle(r), URL: url, Browser: true}) + _ = srv.Close() + if err := <-srvErr; err != nil && err != http.ErrServerClosed { + fmt.Fprintln(os.Stderr, "kage:", err) + return 1 + } + return 0 +} + +// archiveTitle returns the archive's M/Title metadata for use as a window +// title, falling back to the empty string (viewer defaults it to "kage"). +func archiveTitle(r *zim.Reader) string { + if b, err := r.Get(zim.NamespaceMetadata, "Title"); err == nil { + return string(b.Data) + } + return "" +} diff --git a/cli/serve.go b/cli/serve.go new file mode 100644 index 0000000..78fcfdd --- /dev/null +++ b/cli/serve.go @@ -0,0 +1,75 @@ +package cli + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/spf13/cobra" +) + +func newServeCmd() *cobra.Command { + var addr string + cmd := &cobra.Command{ + Use: "serve [dir]", + Short: "Preview a cloned folder in your browser", + Long: "serve runs a local static file server over a cloned folder so you can click\n" + + "through the mirror exactly as a visitor would. With no dir it serves the\n" + + "current directory.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := "." + if len(args) == 1 { + dir = args[0] + } + return runServe(cmd.Context(), dir, addr) + }, + } + cmd.Flags().StringVarP(&addr, "addr", "a", "127.0.0.1:8800", "address to listen on") + return cmd +} + +func runServe(ctx context.Context, dir, addr string) error { + info, err := os.Stat(dir) + if err != nil { + return fmt.Errorf("cannot serve %q: %w", dir, err) + } + if !info.IsDir() { + return fmt.Errorf("%q is not a directory", dir) + } + abs, _ := filepath.Abs(dir) + + ln, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("cannot listen on %s: %w", addr, err) + } + + srv := &http.Server{Handler: http.FileServer(http.Dir(abs))} + fmt.Fprintln(os.Stderr, styleTitle.Render("kage serve")+" "+styleDim.Render(abs)) + fmt.Fprintln(os.Stderr, " open "+styleAccent.Render("http://"+ln.Addr().String())) + fmt.Fprintln(os.Stderr, styleDim.Render(" press Ctrl-C to stop")) + + srvErr := make(chan error, 1) + go func() { srvErr <- srv.Serve(ln) }() + + select { + case err := <-srvErr: + if err != nil && err != http.ErrServerClosed { + return err + } + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + _ = srv.Close() + } + if err := <-srvErr; err != nil && err != http.ErrServerClosed { + return err + } + } + return nil +} diff --git a/cli/styles.go b/cli/styles.go new file mode 100644 index 0000000..18af42a --- /dev/null +++ b/cli/styles.go @@ -0,0 +1,14 @@ +package cli + +import "charm.land/lipgloss/v2" + +// Styles for the human-readable progress and summary lines. They degrade to +// plain text when the terminal has no colour profile. +var ( + styleTitle = lipgloss.NewStyle().Bold(true) + styleAccent = lipgloss.NewStyle().Foreground(lipgloss.Color("12")) + styleOK = lipgloss.NewStyle().Foreground(lipgloss.Color("10")) + styleWarn = lipgloss.NewStyle().Foreground(lipgloss.Color("11")) + styleErr = lipgloss.NewStyle().Foreground(lipgloss.Color("9")) + styleDim = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) +) diff --git a/cli/version.go b/cli/version.go new file mode 100644 index 0000000..c97e1bf --- /dev/null +++ b/cli/version.go @@ -0,0 +1,9 @@ +package cli + +// Build metadata, stamped via -ldflags at release time. goreleaser targets +// github.com/tamnd/kage/cli.{Version,Commit,Date}. +var ( + Version = "dev" + Commit = "none" + Date = "unknown" +) diff --git a/clone/cloner.go b/clone/cloner.go new file mode 100644 index 0000000..2940e77 --- /dev/null +++ b/clone/cloner.go @@ -0,0 +1,570 @@ +package clone + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/tamnd/kage/asset" + "github.com/tamnd/kage/browser" + "github.com/tamnd/kage/robots" + "github.com/tamnd/kage/sanitize" + "github.com/tamnd/kage/urlx" + "golang.org/x/net/html" + "golang.org/x/time/rate" +) + +// Logf is an optional sink for human-readable progress lines. +type Logf func(format string, args ...any) + +// Cloner runs one clone. Build it with New, then call Run. +type Cloner struct { + cfg Config + seed *url.URL + seedHost string + outRoot string + statePth string + + pool *browser.Pool + dl *asset.Downloader + httpC *http.Client + robots *robots.Matcher + front *frontier + stats stats + logf Logf + + mu sync.Mutex + seenAssets map[string]bool + enqueued int // pages offered to the queue + + crawlLimiter *rate.Limiter + + wg sync.WaitGroup + pageJobs chan pageItem + assetJobs chan assetItem + + muContent sync.Mutex + seenContent map[string]string // sha-256 of page bytes -> first path written +} + +type pageItem struct { + u *url.URL + depth int +} + +type assetItem struct { + u *url.URL + referer string +} + +// New builds a Cloner for seed under cfg. It does not touch the network until +// Run is called. +func New(seed *url.URL, cfg Config, logf Logf) *Cloner { + if logf == nil { + logf = func(string, ...any) {} + } + host := seed.Hostname() + outRoot := cfg.HostDir(host) + return &Cloner{ + cfg: cfg, + seed: seed, + seedHost: host, + outRoot: outRoot, + statePth: filepath.Join(outRoot, cfg.Reserved, "state.json"), + dl: asset.NewDownloader(cfg.UserAgent, cfg.Timeout, cfg.MaxAssetBytes), + httpC: &http.Client{Timeout: cfg.Timeout}, + robots: robots.AllowAll(), + front: newFrontier(), + logf: logf, + seenAssets: map[string]bool{}, + seenContent: map[string]string{}, + pageJobs: make(chan pageItem), + assetJobs: make(chan assetItem), + } +} + +// Snapshot returns the current progress, for a CLI ticker. +func (c *Cloner) Snapshot() Progress { return c.stats.snapshot() } + +// pageKey is the dedup identity for a page: its output file. Two URLs that would +// write to the same file (http vs https, "/" vs "/index.html", a trailing +// slash) are the same page and must be crawled only once. +func (c *Cloner) pageKey(u *url.URL) string { + return urlx.LocalPath(c.seedHost, u, urlx.Page, c.cfg.Reserved) +} + +// assetKey is the dedup identity for an asset: its output file, so the same +// bytes referenced over http and https download once. +func (c *Cloner) assetKey(u *url.URL) string { + return urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved) +} + +// pagePathKey is the identity of a page ignoring its query string, used to tell +// a real page apart from its ?q=…/?page=… variants for the progress display. +// Each variant writes its own file (so the crawl stays complete), but they all +// fold to one path here. +func (c *Cloner) pagePathKey(u *url.URL) string { + if u.RawQuery == "" { + return c.pageKey(u) + } + cp := *u + cp.RawQuery = "" + return c.pageKey(&cp) +} + +// Run executes the clone until the frontier drains, MaxPages is hit, or ctx is +// cancelled (which flushes the resume state). It returns the final Result. +func (c *Cloner) Run(ctx context.Context) (Result, error) { + if c.cfg.Force { + _ = os.RemoveAll(c.outRoot) + } + // Refresh re-renders every page in place, so it deliberately skips loading + // the prior visited set; everything else resumes from where it left off. + if c.cfg.Resume && !c.cfg.Refresh { + if err := c.front.load(c.statePth); err != nil { + c.logf("resume: could not load state: %v", err) + } else if n := c.front.visitedCount(); n > 0 { + c.logf("resume: %d pages already done", n) + } + } + + c.pool = browser.New(browser.Options{ + Headless: c.cfg.Headless, + Workers: c.cfg.BrowserPages, + Settle: c.cfg.Settle, + RenderTimeout: c.cfg.RenderTimeout, + Scroll: c.cfg.Scroll, + ChromeBin: c.cfg.ChromeBin, + ControlURL: c.cfg.ControlURL, + }) + defer func() { _ = c.pool.Close() }() + + c.loadRobots(ctx) + c.setupCrawlDelayLimiter() + + // Start workers. + var workers sync.WaitGroup + for range max1(c.cfg.Workers) { + workers.Go(func() { + for j := range c.pageJobs { + c.processPage(ctx, j) + c.wg.Done() + } + }) + } + for range max1(c.cfg.AssetWorkers) { + workers.Go(func() { + for j := range c.assetJobs { + c.processAsset(ctx, j) + c.wg.Done() + } + }) + } + + // Seed. + c.enqueuePage(ctx, c.seed, 0) + if c.cfg.FollowSitemap { + c.seedSitemaps(ctx) + } + + // Close the job channels once every outstanding item is processed. + go func() { + c.wg.Wait() + close(c.pageJobs) + close(c.assetJobs) + }() + workers.Wait() + + if c.cfg.Persist { + if err := c.front.save(c.statePth); err != nil { + c.logf("could not save resume state: %v", err) + } + } + + res := Result{Progress: c.stats.snapshot(), OutDir: c.outRoot, Failures: c.stats.recordedFailures()} + if ctx.Err() != nil { + return res, ctx.Err() + } + return res, nil +} + +// loadRobots fetches and parses robots.txt (unless disabled) and seeds the +// sitemap list it advertises. +func (c *Cloner) loadRobots(ctx context.Context) { + if !c.cfg.RespectRobots { + return + } + robotsURL := c.seed.Scheme + "://" + c.seed.Host + "/robots.txt" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, robotsURL, nil) + if err != nil { + return + } + req.Header.Set("User-Agent", c.cfg.UserAgent) + resp, err := c.httpC.Do(req) + if err != nil { + return + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return + } + data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return + } + c.robots = robots.Parse(string(data), "kage") +} + +func (c *Cloner) setupCrawlDelayLimiter() { + delay := c.cfg.CrawlDelay + if delay <= 0 && c.cfg.RespectRobots && c.robots != nil { + delay = c.robots.CrawlDelay + } + if delay <= 0 { + c.crawlLimiter = nil + return + } + + c.crawlLimiter = rate.NewLimiter(rate.Every(delay), 1) +} + +// seedSitemaps adds in-scope sitemap URLs (from robots and the default path) to +// the frontier. +func (c *Cloner) seedSitemaps(ctx context.Context) { + seeds := append([]string{}, c.robots.Sitemaps...) + seeds = append(seeds, c.seed.Scheme+"://"+c.seed.Host+"/sitemap.xml") + locs := collectSitemaps(ctx, c.httpC, c.cfg.UserAgent, seeds) + added := 0 + for _, loc := range locs { + u, err := urlx.Normalize(c.seed, loc) + if err != nil { + continue + } + if urlx.InScope(c.seed, u, c.cfg.scope()) && urlx.LikelyPage(u) { + if c.enqueuePage(ctx, u, 1) { + added++ + } + } + } + if added > 0 { + c.logf("sitemap: seeded %d URLs", added) + } +} + +// processPage renders one page, rewrites its references to local paths, strips +// every script, and writes the result. +func (c *Cloner) processPage(ctx context.Context, j pageItem) { + if ctx.Err() != nil { + return + } + key := c.pageKey(j.u) + if c.cfg.RespectRobots && !c.robots.Allowed(j.u.Path) { + c.stats.skipped.Add(1) + return + } + if !c.waitForCrawlDelay(ctx) { + return + } + + res, err := c.pool.Render(ctx, j.u.String()) + if err != nil { + var notHTML *browser.ErrNotHTML + if errors.As(err, ¬HTML) { + // The URL is not a page but a file (a zip, a CSV, a bare image) that + // reached the page worker through an extensionless link. Hand it to the + // asset downloader, where the size and media policy decides whether to + // localise it or leave it remote, rather than saving a broken page or + // letting Chrome download it to the user's Downloads folder (issue #32). + c.front.markVisited(key) + if c.wantAsset(j.u) { + c.enqueueAsset(ctx, j.u, "") + c.logf("not a page, fetching as asset (%s): %s", notHTML.ContentType, j.u.String()) + } else { + c.logf("not a page, left on the live web (%s): %s", notHTML.ContentType, j.u.String()) + } + return + } + c.failPage(j.u.String(), fmt.Errorf("render: %w", err)) + return + } + + root, err := html.Parse(strings.NewReader(res.HTML)) + if err != nil { + c.failPage(j.u.String(), fmt.Errorf("parse: %w", err)) + return + } + + localFile := urlx.LocalPath(c.seedHost, j.u, urlx.Page, c.cfg.Reserved) + fileDir := urlx.Dir(localFile) + + sink := func(u *url.URL, kind urlx.Kind) string { + switch kind { + case urlx.Page: + if urlx.InScope(c.seed, u, c.cfg.scope()) { + c.enqueuePage(ctx, u, j.depth+1) + local := urlx.LocalPath(c.seedHost, u, urlx.Page, c.cfg.Reserved) + return urlx.Rel(fileDir, local) + } + return u.String() // external page link stays on the live web + default: // Asset + if !c.wantAsset(u) { + return u.String() // off-domain or bulk media: leave it on the live web + } + c.enqueueAsset(ctx, u, j.u.String()) + local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved) + return urlx.Rel(fileDir, local) + } + } + + asset.RewriteHTML(root, j.u, sink) + sanitize.CleanTree(root, sanitize.Options{ + KeepNoscript: c.cfg.KeepNoscript, + MobileReadable: c.cfg.MobileReadable, + Banner: "cloned by kage from " + j.u.String(), + }) + + var buf strings.Builder + if err := html.Render(&buf, root); err != nil { + c.failPage(j.u.String(), fmt.Errorf("render html: %w", err)) + return + } + deduped, err := c.writePage(localFile, []byte(buf.String())) + if err != nil { + c.failPage(j.u.String(), fmt.Errorf("write %s: %w", localFile, err)) + return + } + c.front.markVisited(key) + c.stats.recordPage(c.pagePathKey(j.u), deduped) +} + +// waitForCrawlDelay spaces page render starts. +func (c *Cloner) waitForCrawlDelay(ctx context.Context) bool { + if c.crawlLimiter == nil { + return true + } + + return c.crawlLimiter.Wait(ctx) == nil +} + +// processAsset downloads one asset, rewriting CSS references on the way, and +// writes it to its deterministic local path. +func (c *Cloner) processAsset(ctx context.Context, j assetItem) { + if ctx.Err() != nil { + return + } + res, err := c.dl.Get(ctx, j.u, j.referer) + if err != nil { + if errors.Is(err, asset.ErrTooLarge) { + // Over the size cap: leave it out rather than save a truncated + // fragment. Count it as skipped, not failed, so the run is clean. + c.stats.assetSkipped.Add(1) + c.logf("asset skipped (over %d MB): %s", c.cfg.MaxAssetBytes>>20, j.u.String()) + return + } + c.failAsset(j.u.String(), j.referer, err) + return + } + + localFile := urlx.LocalPath(c.seedHost, j.u, urlx.Asset, c.cfg.Reserved) + body := res.Body + if res.IsCSS { + fileDir := urlx.Dir(localFile) + cssSink := func(u *url.URL, _ urlx.Kind) string { + if !c.wantAsset(u) { + return u.String() // off-domain or bulk media: leave it on the live web + } + c.enqueueAsset(ctx, u, j.u.String()) + local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved) + return urlx.Rel(fileDir, local) + } + body = asset.RewriteCSS(body, j.u, cssSink) + } + if err := c.writeFile(localFile, body); err != nil { + c.failAsset(j.u.String(), j.referer, fmt.Errorf("write %s: %w", localFile, err)) + return + } + c.stats.assets.Add(1) +} + +// failAsset records and logs a failed asset, naming the page that referenced it +// so a 403 or 404 is traceable back to where it came from. The reason is +// classified (HTTP status, timeout, or other) for a readable line. +func (c *Cloner) failAsset(u, referer string, err error) { + c.stats.assetErrors.Add(1) + reason := classifyError(err) + c.stats.recordFailure(Failure{Kind: "asset", URL: u, Referer: referer, Reason: reason}) + if referer != "" { + c.logf("asset error: %s\n %s\n referenced by %s", reason, u, referer) + } else { + c.logf("asset error: %s\n %s", reason, u) + } +} + +// failPage records and logs a failed page. +func (c *Cloner) failPage(u string, err error) { + c.stats.pageErrors.Add(1) + reason := classifyError(err) + c.stats.recordFailure(Failure{Kind: "page", URL: u, Reason: reason}) + c.logf("page error: %s\n %s", reason, u) +} + +// classifyError turns an error into a short, human-readable reason for the log +// and the final report: an HTTP status with its name, a timeout, a cancellation, +// or the underlying message otherwise. +func classifyError(err error) string { + if err == nil { + return "" + } + var se *asset.StatusError + if errors.As(err, &se) { + return se.Error() + } + switch { + case errors.Is(err, context.DeadlineExceeded): + return "timed out" + case errors.Is(err, context.Canceled): + return "cancelled" + } + return err.Error() +} + +// enqueuePage offers a page URL to the frontier, honouring the visited set, the +// depth cap, and the page budget. It reports whether the page was newly queued. +func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool { + if c.cfg.MaxDepth > 0 && depth > c.cfg.MaxDepth { + return false + } + key := c.pageKey(u) + if c.front.isVisited(key) { + return false + } + if !c.front.offer(key) { + return false + } + c.mu.Lock() + if c.cfg.MaxPages > 0 && c.enqueued >= c.cfg.MaxPages { + c.mu.Unlock() + return false + } + c.enqueued++ + c.mu.Unlock() + + c.wg.Add(1) + go func() { + select { + case c.pageJobs <- pageItem{u: u, depth: depth}: + case <-ctx.Done(): + c.wg.Done() + } + }() + return true +} + +// wantAsset reports whether an asset should be downloaded and localized, or left +// pointing at its live URL. kage skips two classes by default: assets on a host +// outside the seed's registrable domain (a third-party tracker, an unrelated +// CDN), and bulk media, installers, and archives whose extension is in the skip +// set. Both are decisions a page worker can make from the URL alone, before any +// download, so the rewritten HTML simply keeps the remote link. +func (c *Cloner) wantAsset(u *url.URL) bool { + if c.cfg.AssetSameDomain && !urlx.SameRegistrableDomain(c.seed, u) { + return false + } + if c.cfg.SkipAssetExts[urlx.Ext(u)] { + return false + } + return true +} + +// enqueueAsset offers an asset URL for download, deduping by canonical URL. +func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) { + key := c.assetKey(u) + c.mu.Lock() + if c.seenAssets[key] { + c.mu.Unlock() + return + } + c.seenAssets[key] = true + c.mu.Unlock() + + c.wg.Add(1) + go func() { + select { + case c.assetJobs <- assetItem{u: u, referer: referer}: + case <-ctx.Done(): + c.wg.Done() + } + }() +} + +// writePage writes a rendered page, deduping by content. The first page with a +// given byte content is written normally; a later page with identical bytes is +// stored as a hard link to that first file, so the same content never occupies +// disk twice (a faceted site whose ?q=… variants all render the same page is the +// motivating case). It reports whether this write was deduped. If hard links are +// unsupported, it falls back to writing the bytes, so correctness never depends +// on the link succeeding. +func (c *Cloner) writePage(relSlash string, data []byte) (bool, error) { + sum := sha256.Sum256(data) + h := string(sum[:]) + + c.muContent.Lock() + canon, seen := c.seenContent[h] + if !seen { + c.seenContent[h] = relSlash + } + c.muContent.Unlock() + + if seen && canon != relSlash { + if err := c.linkFile(canon, relSlash); err == nil { + return true, nil + } + // The canonical file may not be on disk yet (a concurrent first write) + // or the filesystem may not support links; write the bytes instead. + } + return false, c.writeFile(relSlash, data) +} + +// linkFile hard-links the already-written canonical file to target, replacing +// any file already at target. Both paths are confined to the mirror root. +func (c *Cloner) linkFile(canonSlash, targetSlash string) error { + canonFull := filepath.Join(c.outRoot, filepath.FromSlash(canonSlash)) + targetFull := filepath.Join(c.outRoot, filepath.FromSlash(targetSlash)) + if !strings.HasPrefix(targetFull, filepath.Clean(c.outRoot)+string(os.PathSeparator)) { + return fmt.Errorf("refusing to link outside mirror root: %s", targetSlash) + } + if err := os.MkdirAll(filepath.Dir(targetFull), 0o755); err != nil { + return err + } + _ = os.Remove(targetFull) + return os.Link(canonFull, targetFull) +} + +// writeFile writes data to a slash path relative to the mirror root, creating +// parent directories. The path is cleaned so it can never escape the root. +func (c *Cloner) writeFile(relSlash string, data []byte) error { + rel := filepath.FromSlash(relSlash) + full := filepath.Join(c.outRoot, rel) + if !strings.HasPrefix(full, filepath.Clean(c.outRoot)+string(os.PathSeparator)) && full != c.outRoot { + return fmt.Errorf("refusing to write outside mirror root: %s", relSlash) + } + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + return err + } + return os.WriteFile(full, data, 0o644) +} + +func max1(n int) int { + if n < 1 { + return 1 + } + return n +} diff --git a/clone/cloner_test.go b/clone/cloner_test.go new file mode 100644 index 0000000..5ef038f --- /dev/null +++ b/clone/cloner_test.go @@ -0,0 +1,425 @@ +package clone + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/tamnd/kage/browser" + "github.com/tamnd/kage/robots" + "github.com/tamnd/kage/urlx" +) + +// testSite is a tiny two-page site with a stylesheet, an image, an inline +// script, an onclick handler, and a javascript: link, so a full clone exercises +// rendering, asset localisation, and JavaScript stripping at once. +func testSite(t *testing.T) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(`<!doctype html><html><head> +<link rel="stylesheet" href="/site.css"> +<script src="/app.js"></script> +</head><body> +<h1>Home</h1> +<img src="/logo.png" alt="logo"> +<a href="/about">About</a> +<a href="javascript:void(0)" onclick="boom()">Danger</a> +<script>console.log("inline")</script> +</body></html>`)) + }) + mux.HandleFunc("/about", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(`<!doctype html><html><head> +<link rel="stylesheet" href="/site.css"> +</head><body><h1>About</h1><a href="/">Home</a></body></html>`)) + }) + mux.HandleFunc("/site.css", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/css") + _, _ = w.Write([]byte(`body{background:url("/bg.png")} h1{color:red}`)) + }) + mux.HandleFunc("/app.js", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/javascript") + _, _ = w.Write([]byte(`document.body.dataset.ran = "1";`)) + }) + mux.HandleFunc("/logo.png", servePNG) + mux.HandleFunc("/bg.png", servePNG) + mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("User-agent: *\nAllow: /\n")) + }) + return httptest.NewServer(mux) +} + +// a 1x1 transparent PNG. +var pngBytes = []byte{ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, + 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89, 0x00, 0x00, 0x00, + 0x0a, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0x00, 0x01, 0x00, 0x00, + 0x05, 0x00, 0x01, 0x0d, 0x0a, 0x2d, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x49, + 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, +} + +func servePNG(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(pngBytes) +} + +func TestCloneEndToEnd(t *testing.T) { + if testing.Short() { + t.Skip("clone end-to-end drives Chrome; skipped under -short") + } + if _, ok := browser.LookChrome(); !ok { + t.Skip("no Chrome/Chromium found; skipping clone end-to-end") + } + + srv := testSite(t) + defer srv.Close() + + seed, err := urlx.ParseSeed(srv.URL) + if err != nil { + t.Fatalf("parse seed: %v", err) + } + + out := t.TempDir() + cfg := DefaultConfig() + cfg.OutDir = out + cfg.Settle = 300 * time.Millisecond + cfg.RenderTimeout = 20 * time.Second + cfg.Timeout = 10 * time.Second + + c := New(seed, cfg, t.Logf) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + res, err := c.Run(ctx) + if err != nil { + t.Fatalf("run: %v", err) + } + if res.Pages < 2 { + t.Fatalf("expected at least 2 pages, got %d", res.Pages) + } + + root := res.OutDir + indexPath := filepath.Join(root, "index.html") + body := readFile(t, indexPath) + + // JavaScript is gone: no <script>, no onclick, no javascript: URL. + if strings.Contains(strings.ToLower(body), "<script") { + t.Error("index.html still contains a <script> tag") + } + if strings.Contains(strings.ToLower(body), "onclick") { + t.Error("index.html still contains an onclick handler") + } + if strings.Contains(strings.ToLower(body), "javascript:") { + t.Error("index.html still contains a javascript: URL") + } + + // Layout is preserved: the stylesheet link survives and points local. + if !strings.Contains(body, "stylesheet") { + t.Error("index.html lost its stylesheet link") + } + if strings.Contains(body, srv.URL+"/site.css") { + t.Error("stylesheet still points at the live origin") + } + + // The about page and the localised assets exist on disk. + if !fileExists(filepath.Join(root, "about", "index.html")) { + t.Error("about page was not written") + } + assetDir := filepath.Join(root, cfg.Reserved) + if !anyFileUnder(t, assetDir, "site.css") { + t.Error("site.css was not downloaded") + } + if !anyFileUnder(t, assetDir, "logo.png") { + t.Error("logo.png was not downloaded") + } + + // The localised CSS had its url() rewritten away from the origin. + css := readAnyFile(t, assetDir, "site.css") + if strings.Contains(css, srv.URL) { + t.Error("site.css still references the live origin in url()") + } +} + +// TestPageKeyCollapsesDuplicates guards the dedup identity: http vs https, a +// trailing slash, and "/index.html" vs "/" all name the same output file, so a +// page is crawled once rather than two-to-four times. +func TestPageKeyCollapsesDuplicates(t *testing.T) { + seed, _ := urlx.ParseSeed("https://ex.com") + c := New(seed, DefaultConfig(), nil) + + groups := [][]string{ + {"https://ex.com/", "http://ex.com/", "https://ex.com/index.html", "http://ex.com/index.html"}, + {"https://ex.com/docs", "http://ex.com/docs", "https://ex.com/docs/", "https://ex.com/docs/index.html"}, + } + for _, g := range groups { + want := c.pageKey(mustURL(t, g[0])) + for _, raw := range g[1:] { + if got := c.pageKey(mustURL(t, raw)); got != want { + t.Errorf("pageKey(%q) = %q, want %q (same page)", raw, got, want) + } + } + } + + // Genuinely different pages must not collapse. + if c.pageKey(mustURL(t, "https://ex.com/a")) == c.pageKey(mustURL(t, "https://ex.com/b")) { + t.Error("distinct pages share a key") + } +} + +func TestCrawlDelaySpacesPageStarts(t *testing.T) { + seed, _ := urlx.ParseSeed("https://ex.com") + cfg := DefaultConfig() + cfg.RespectRobots = true + c := New(seed, cfg, nil) + c.robots = &robots.Matcher{CrawlDelay: 20 * time.Millisecond} + c.setupCrawlDelayLimiter() + + ctx := context.Background() + if !c.waitForCrawlDelay(ctx) { + t.Fatal("first crawl-delay wait returned false") + } + + start := time.Now() + if !c.waitForCrawlDelay(ctx) { + t.Fatal("second crawl-delay wait returned false") + } + if elapsed := time.Since(start); elapsed < 15*time.Millisecond { + t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed) + } +} + +func TestCrawlDelayFlagOverridesRobots(t *testing.T) { + seed, _ := urlx.ParseSeed("https://ex.com") + cfg := DefaultConfig() + cfg.RespectRobots = true + cfg.CrawlDelay = 20 * time.Millisecond + c := New(seed, cfg, nil) + c.robots = &robots.Matcher{CrawlDelay: time.Minute} + c.setupCrawlDelayLimiter() + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + if !c.waitForCrawlDelay(ctx) { + t.Fatal("first crawl-delay wait returned false") + } + + start := time.Now() + if !c.waitForCrawlDelay(ctx) { + t.Fatal("second crawl-delay wait returned false") + } + elapsed := time.Since(start) + if elapsed < 15*time.Millisecond { + t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed) + } + if elapsed > 150*time.Millisecond { + t.Fatalf("second crawl-delay wait = %v, override likely ignored", elapsed) + } +} + +func mustURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse %q: %v", raw, err) + } + return u +} + +func TestCloneResumeSkipsVisited(t *testing.T) { + if testing.Short() { + t.Skip("resume test drives Chrome; skipped under -short") + } + if _, ok := browser.LookChrome(); !ok { + t.Skip("no Chrome/Chromium found; skipping resume test") + } + + srv := testSite(t) + defer srv.Close() + seed, _ := urlx.ParseSeed(srv.URL) + + out := t.TempDir() + cfg := DefaultConfig() + cfg.OutDir = out + cfg.Settle = 300 * time.Millisecond + + c1 := New(seed, cfg, t.Logf) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + if _, err := c1.Run(ctx); err != nil { + t.Fatalf("first run: %v", err) + } + + // Second run with resume on should find the state and re-render nothing new. + c2 := New(seed, cfg, t.Logf) + res2, err := c2.Run(ctx) + if err != nil { + t.Fatalf("second run: %v", err) + } + if res2.Pages != 0 { + t.Fatalf("resume should skip all visited pages, but rendered %d", res2.Pages) + } +} + +func TestCloneRefreshReRenders(t *testing.T) { + if testing.Short() { + t.Skip("refresh test drives Chrome; skipped under -short") + } + if _, ok := browser.LookChrome(); !ok { + t.Skip("no Chrome/Chromium found; skipping refresh test") + } + + srv := testSite(t) + defer srv.Close() + seed, _ := urlx.ParseSeed(srv.URL) + + out := t.TempDir() + cfg := DefaultConfig() + cfg.OutDir = out + cfg.Settle = 300 * time.Millisecond + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + if _, err := New(seed, cfg, t.Logf).Run(ctx); err != nil { + t.Fatalf("first run: %v", err) + } + + // Resume keeps everything in place but renders nothing new; refresh, on the + // same mirror, re-renders every page so changed content is pulled back in. + cfg.Refresh = true + res, err := New(seed, cfg, t.Logf).Run(ctx) + if err != nil { + t.Fatalf("refresh run: %v", err) + } + if res.Pages < 2 { + t.Fatalf("refresh should re-render every page, got %d", res.Pages) + } + if !fileExists(filepath.Join(out, seed.Hostname(), "index.html")) { + t.Error("refresh removed the mirror instead of overwriting in place") + } +} + +// TestCloneRoutesNonHTMLToAsset guards issue #32: an extensionless link that +// turns out to be a file (a zip) is classified as a page up front, but once the +// page worker sees it is not HTML it must be handed to the asset downloader, not +// saved as a broken page nor downloaded by Chrome to ~/Downloads. +func TestCloneRoutesNonHTMLToAsset(t *testing.T) { + if testing.Short() { + t.Skip("clone test drives Chrome; skipped under -short") + } + if _, ok := browser.LookChrome(); !ok { + t.Skip("no Chrome/Chromium found; skipping clone test") + } + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + // The link has no extension, so it is queued as a page; the server then + // answers it with a zip. + _, _ = w.Write([]byte(`<!doctype html><html><body> +<h1>Home</h1><a href="/download">grab the bundle</a></body></html>`)) + }) + mux.HandleFunc("/download", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write([]byte("PK\x03\x04 pretend bundle")) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + seed, err := urlx.ParseSeed(srv.URL) + if err != nil { + t.Fatalf("parse seed: %v", err) + } + + out := t.TempDir() + cfg := DefaultConfig() + cfg.OutDir = out + cfg.Settle = 300 * time.Millisecond + cfg.RenderTimeout = 20 * time.Second + cfg.Timeout = 10 * time.Second + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + res, err := New(seed, cfg, t.Logf).Run(ctx) + if err != nil { + t.Fatalf("run: %v", err) + } + + root := res.OutDir + // The home page is a real page and is written. + if !fileExists(filepath.Join(root, "index.html")) { + t.Error("home page was not written") + } + // The zip is NOT saved as a page: no download/index.html exists. + if fileExists(filepath.Join(root, "download", "index.html")) { + t.Error("non-HTML target was saved as a page") + } + // The zip is fetched as an asset under the reserved tree instead. + if res.Assets < 1 { + t.Errorf("expected the zip to be fetched as an asset, assets=%d", res.Assets) + } + assetDir := filepath.Join(root, cfg.Reserved) + if !anyFileUnder(t, assetDir, "download") { + t.Error("the zip was not downloaded into the reserved asset tree") + } + if res.PageErrors != 0 { + t.Errorf("a rerouted non-HTML target must not count as a page error, got %d", res.PageErrors) + } +} + +func readFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(b) +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func anyFileUnder(t *testing.T, dir, name string) bool { + t.Helper() + found := false + _ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() && strings.HasSuffix(p, name) { + found = true + } + return nil + }) + return found +} + +func readAnyFile(t *testing.T, dir, name string) string { + t.Helper() + var out string + _ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() && strings.HasSuffix(p, name) { + b, _ := os.ReadFile(p) + out = string(b) + } + return nil + }) + return out +} diff --git a/clone/config.go b/clone/config.go new file mode 100644 index 0000000..97d7eed --- /dev/null +++ b/clone/config.go @@ -0,0 +1,146 @@ +// Package clone is kage's engine: it ties the Chrome pool, the JavaScript +// stripper, the asset localiser, and the URL↔path mapper into one resumable, +// polite crawl that turns a live site into a browsable offline folder. +package clone + +import ( + "os" + "path/filepath" + "time" + + "github.com/tamnd/kage/urlx" +) + +// DefaultOutDir is where mirrors land unless --out overrides it: a per-user data +// directory ($HOME/data/kage) so clones from anywhere collect in one place, +// falling back to a local kage-out when the home directory cannot be resolved. +func DefaultOutDir() string { + home, err := os.UserHomeDir() + if err != nil || home == "" { + return "kage-out" + } + return filepath.Join(home, "data", "kage") +} + +// Config is the full set of knobs for a clone run. DefaultConfig fills the +// baseline; the CLI overlays flags on top. +type Config struct { + OutDir string // output root; the mirror lands in <OutDir>/<host>/ + Reserved string // reserved dir name for assets and state (default "_kage") + + Workers int // page render workers + AssetWorkers int // HTTP asset download workers + BrowserPages int // Chrome page-pool size + MaxPages int // stop after N pages (0 = unlimited) + MaxDepth int // BFS/DFS depth cap (0 = unlimited) + Traversal string + MaxAssetBytes int64 + + // AssetSameDomain, when set, localizes only assets whose host shares the + // seed's registrable domain (apple.com covers developer.apple.com and + // www.apple.com but not cdn-apple.com or an unrelated third party). An + // off-domain asset is left pointing at its live URL instead of downloaded. + AssetSameDomain bool + // SkipAssetExts lists asset extensions (".mp4", ".pdf", ".dmg", …) that are + // left on the live web rather than downloaded, so bulk media, installers, and + // archives do not bloat the mirror. The reference keeps its remote URL. + SkipAssetExts map[string]bool + + Timeout time.Duration // per HTTP request + Settle time.Duration // network-idle quiet period + RenderTimeout time.Duration // hard cap per page render + Scroll bool + + UserAgent string + IncludeSubdomains bool + ScopePrefix string + ExcludePaths []string + + RespectRobots bool + CrawlDelay time.Duration // override robots.txt Crawl-delay when > 0 + FollowSitemap bool + Headless bool + KeepNoscript bool + MobileReadable bool + ChromeBin string + ControlURL string + + // Resume loads the prior run's visited set and skips pages already written, + // so an interrupted or repeated clone picks up where it left off instead of + // refetching. Refresh forces every page to be re-rendered in place (the + // mirror is kept, files are overwritten) to pull in changed content. Force + // deletes the mirror first for a clean-slate clone. Persist writes the + // visited set back to state.json when the run ends. + Resume bool + Refresh bool + Force bool + Persist bool +} + +// DefaultUserAgent is a current desktop Chrome UA, used by the asset fetcher and +// the robots fetch so a site treats kage like the browser it drives. +const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" + +// DefaultSkipAssetExts returns the asset extensions kage leaves on the live web +// by default: bulk media, installers, and archives that rarely matter for +// reading a site offline but dominate its download size (a docs site's WWDC +// videos, .dmg/.pkg installers, and PDF manuals can be most of the bytes). +// Page-rendering assets (images, fonts, CSS) are deliberately absent, so the +// offline pages still look right. +func DefaultSkipAssetExts() map[string]bool { + exts := []string{ + // Video and audio. + ".mp4", ".m4v", ".mov", ".avi", ".mkv", ".webm", ".flv", ".wmv", + ".m3u8", ".ts", ".mp3", ".wav", ".flac", ".aac", ".ogg", ".oga", + // Installers and disk images. + ".dmg", ".pkg", ".exe", ".msi", ".deb", ".rpm", ".appimage", ".iso", + // Archives. + ".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".7z", ".rar", + // Documents that download rather than render. + ".pdf", + } + m := make(map[string]bool, len(exts)) + for _, e := range exts { + m[e] = true + } + return m +} + +// DefaultConfig returns the baseline configuration. +func DefaultConfig() Config { + return Config{ + OutDir: DefaultOutDir(), + Reserved: urlx.DefaultReserved, + Workers: 4, + AssetWorkers: 8, + BrowserPages: 4, + MaxAssetBytes: 25 << 20, + AssetSameDomain: true, + SkipAssetExts: DefaultSkipAssetExts(), + Traversal: "bfs", + Timeout: 30 * time.Second, + Settle: 1500 * time.Millisecond, + RenderTimeout: 30 * time.Second, + UserAgent: DefaultUserAgent, + RespectRobots: true, + FollowSitemap: true, + Headless: true, + Resume: true, + Persist: true, + } +} + +// HostDir returns the mirror root for a seed host: <OutDir>/<host>. +func (c Config) HostDir(host string) string { + return filepath.Join(c.OutDir, host) +} + +// scope builds the urlx scope config from the run config. +func (c Config) scope() urlx.ScopeConfig { + return urlx.ScopeConfig{ + IncludeSubdomains: c.IncludeSubdomains, + ScopePrefix: c.ScopePrefix, + ExcludePaths: c.ExcludePaths, + } +} diff --git a/clone/dedup_test.go b/clone/dedup_test.go new file mode 100644 index 0000000..89dca3f --- /dev/null +++ b/clone/dedup_test.go @@ -0,0 +1,69 @@ +package clone + +import ( + "os" + "path/filepath" + "testing" +) + +// TestWritePageDedup checks that identical page bytes are stored once and shared +// by a hard link, while different bytes are written as their own file. +func TestWritePageDedup(t *testing.T) { + dir := t.TempDir() + c := &Cloner{outRoot: dir, seenContent: map[string]string{}} + + body := []byte("<html><body>same page</body></html>") + + if deduped, err := c.writePage("a/index.html", body); err != nil || deduped { + t.Fatalf("first write: deduped=%v err=%v, want false/nil", deduped, err) + } + if deduped, err := c.writePage("b/index.html", body); err != nil || !deduped { + t.Fatalf("second identical write: deduped=%v err=%v, want true/nil", deduped, err) + } + if deduped, err := c.writePage("c/index.html", []byte("<html>other</html>")); err != nil || deduped { + t.Fatalf("third different write: deduped=%v err=%v, want false/nil", deduped, err) + } + + // The two identical pages must be the same file on disk (one inode). + fa, err := os.Stat(filepath.Join(dir, "a/index.html")) + if err != nil { + t.Fatal(err) + } + fb, err := os.Stat(filepath.Join(dir, "b/index.html")) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(fa, fb) { + t.Error("identical pages were not hard-linked to the same file") + } + + // The different page must stand alone with its own bytes. + got, err := os.ReadFile(filepath.Join(dir, "c/index.html")) + if err != nil { + t.Fatal(err) + } + if string(got) != "<html>other</html>" { + t.Errorf("third page content = %q", got) + } +} + +// TestRecordPageCounts checks that pages counts every write, pagePaths counts +// distinct query-stripped paths, and pagesLinked counts deduped writes. +func TestRecordPageCounts(t *testing.T) { + var s stats + s.recordPage("showcase/index.html", false) + s.recordPage("showcase/index.html", true) // a ?q= variant of the same path + s.recordPage("showcase/index.html", true) + s.recordPage("about/index.html", false) + + p := s.snapshot() + if p.Pages != 4 { + t.Errorf("Pages = %d, want 4", p.Pages) + } + if p.PagePaths != 2 { + t.Errorf("PagePaths = %d, want 2", p.PagePaths) + } + if p.PagesLinked != 2 { + t.Errorf("PagesLinked = %d, want 2", p.PagesLinked) + } +} diff --git a/clone/frontier.go b/clone/frontier.go new file mode 100644 index 0000000..af01967 --- /dev/null +++ b/clone/frontier.go @@ -0,0 +1,106 @@ +package clone + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "sync" +) + +// frontier is the deduped set of page URLs kage has already seen. It is small, +// concurrency-safe, and persists to disk so --resume can skip work already done. +// The actual queueing is handled by the cloner's channels; the frontier only +// answers "is this URL new?" and remembers the answer. +type frontier struct { + mu sync.Mutex + seen map[string]bool // queued or visited + visited map[string]bool // fully written +} + +func newFrontier() *frontier { + return &frontier{seen: map[string]bool{}, visited: map[string]bool{}} +} + +// offer reports whether key is new (and records it as seen). A repeated key +// returns false so it is enqueued only once. +func (f *frontier) offer(key string) bool { + f.mu.Lock() + defer f.mu.Unlock() + if f.seen[key] { + return false + } + f.seen[key] = true + return true +} + +// markVisited records that a page was written. +func (f *frontier) markVisited(key string) { + f.mu.Lock() + f.visited[key] = true + f.mu.Unlock() +} + +// isVisited reports whether a page was already written in a previous run. +func (f *frontier) isVisited(key string) bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.visited[key] +} + +func (f *frontier) visitedCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.visited) +} + +// state is the JSON shape persisted for resume. +type state struct { + Visited []string `json:"visited"` +} + +// load reads a previously saved visited set; a missing file is not an error. +func (f *frontier) load(path string) error { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + var s state + if err := json.Unmarshal(data, &s); err != nil { + return err + } + f.mu.Lock() + defer f.mu.Unlock() + for _, v := range s.Visited { + f.visited[v] = true + f.seen[v] = true + } + return nil +} + +// save writes the visited set atomically (write temp, rename). +func (f *frontier) save(path string) error { + f.mu.Lock() + visited := make([]string, 0, len(f.visited)) + for v := range f.visited { + visited = append(visited, v) + } + f.mu.Unlock() + sort.Strings(visited) + + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + data, err := json.MarshalIndent(state{Visited: visited}, "", " ") + if err != nil { + return err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/clone/frontier_test.go b/clone/frontier_test.go new file mode 100644 index 0000000..534ff25 --- /dev/null +++ b/clone/frontier_test.go @@ -0,0 +1,67 @@ +package clone + +import ( + "path/filepath" + "testing" +) + +func TestFrontierOfferDedups(t *testing.T) { + f := newFrontier() + if !f.offer("a") { + t.Fatal("first offer of a should be new") + } + if f.offer("a") { + t.Fatal("second offer of a should be a duplicate") + } + if !f.offer("b") { + t.Fatal("first offer of b should be new") + } +} + +func TestFrontierVisited(t *testing.T) { + f := newFrontier() + if f.isVisited("x") { + t.Fatal("x should not be visited yet") + } + f.markVisited("x") + if !f.isVisited("x") { + t.Fatal("x should be visited after markVisited") + } + if f.visitedCount() != 1 { + t.Fatalf("visitedCount = %d, want 1", f.visitedCount()) + } +} + +func TestFrontierSaveLoadRoundTrip(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sub", "state.json") + + f := newFrontier() + f.markVisited("https://example.com/") + f.markVisited("https://example.com/about") + if err := f.save(path); err != nil { + t.Fatalf("save: %v", err) + } + + g := newFrontier() + if err := g.load(path); err != nil { + t.Fatalf("load: %v", err) + } + if g.visitedCount() != 2 { + t.Fatalf("loaded visitedCount = %d, want 2", g.visitedCount()) + } + if !g.isVisited("https://example.com/about") { + t.Fatal("about should be visited after load") + } + // A loaded visited URL is also seen, so it is not re-offered. + if g.offer("https://example.com/about") { + t.Fatal("a loaded URL should not be offered again") + } +} + +func TestFrontierLoadMissingIsNotError(t *testing.T) { + f := newFrontier() + if err := f.load(filepath.Join(t.TempDir(), "nope.json")); err != nil { + t.Fatalf("loading a missing file should be a no-op, got %v", err) + } +} diff --git a/clone/sitemap.go b/clone/sitemap.go new file mode 100644 index 0000000..d736248 --- /dev/null +++ b/clone/sitemap.go @@ -0,0 +1,83 @@ +package clone + +import ( + "context" + "encoding/xml" + "net/http" + "strings" + "time" +) + +// sitemapDoc covers both a urlset and a sitemapindex: both carry <loc> elements, +// so one shape extracts the URLs from either. +type sitemapDoc struct { + URLs []sitemapEntry `xml:"url"` + Sitemaps []sitemapEntry `xml:"sitemap"` +} + +type sitemapEntry struct { + Loc string `xml:"loc"` +} + +// fetchSitemap downloads and parses one sitemap, returning page locations and, +// for a sitemap index, the child sitemap URLs. +func fetchSitemap(ctx context.Context, client *http.Client, ua, sitemapURL string) (locs, children []string) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, sitemapURL, nil) + if err != nil { + return nil, nil + } + if ua != "" { + req.Header.Set("User-Agent", ua) + } + resp, err := client.Do(req) + if err != nil { + return nil, nil + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, nil + } + var doc sitemapDoc + dec := xml.NewDecoder(resp.Body) + if err := dec.Decode(&doc); err != nil { + return nil, nil + } + for _, u := range doc.URLs { + if s := strings.TrimSpace(u.Loc); s != "" { + locs = append(locs, s) + } + } + for _, sm := range doc.Sitemaps { + if s := strings.TrimSpace(sm.Loc); s != "" { + children = append(children, s) + } + } + return locs, children +} + +// collectSitemaps walks a set of seed sitemap URLs (following index files one +// level deep) and returns all discovered page locations, bounded by a deadline. +func collectSitemaps(ctx context.Context, client *http.Client, ua string, seeds []string) []string { + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + + var out []string + seen := map[string]bool{} + queue := append([]string{}, seeds...) + for len(queue) > 0 && ctx.Err() == nil { + sm := queue[0] + queue = queue[1:] + if seen[sm] { + continue + } + seen[sm] = true + locs, children := fetchSitemap(ctx, client, ua, sm) + out = append(out, locs...) + for _, c := range children { + if !seen[c] { + queue = append(queue, c) + } + } + } + return out +} diff --git a/clone/stats.go b/clone/stats.go new file mode 100644 index 0000000..635ed7d --- /dev/null +++ b/clone/stats.go @@ -0,0 +1,111 @@ +package clone + +import ( + "sync" + "sync/atomic" +) + +// maxRecordedFailures caps how many individual failures Run keeps for the final +// report, so a huge broken site cannot grow the slice without bound. The error +// counters still count every failure. +const maxRecordedFailures = 100 + +// stats are the live counters of a run, read by the CLI's progress ticker. +type stats struct { + pages atomic.Int64 // page documents written (one per output file) + pagePaths atomic.Int64 // distinct URL paths among those, ignoring query + pagesLinked atomic.Int64 // pages stored as a hard link to identical content + assets atomic.Int64 + assetSkipped atomic.Int64 // assets left on the live web (over the size cap) + pageErrors atomic.Int64 + assetErrors atomic.Int64 + skipped atomic.Int64 // robots-disallowed or out of budget + + muPaths sync.Mutex + seenPath map[string]struct{} + + muFail sync.Mutex + failures []Failure +} + +// recordPage counts a freshly written page. Every write bumps pages; the first +// write for a given query-stripped path also bumps pagePaths, so the display can +// separate real pages from the query-string variants (?q=…, ?page=…) that a +// single path can spawn by the thousand on a faceted site. deduped marks a page +// whose bytes were stored as a hard link to identical content already on disk. +func (s *stats) recordPage(pathKey string, deduped bool) { + s.pages.Add(1) + if deduped { + s.pagesLinked.Add(1) + } + s.muPaths.Lock() + if s.seenPath == nil { + s.seenPath = make(map[string]struct{}) + } + if _, ok := s.seenPath[pathKey]; !ok { + s.seenPath[pathKey] = struct{}{} + s.pagePaths.Add(1) + } + s.muPaths.Unlock() +} + +// Failure is one thing that went wrong, kept for the end-of-run report so the +// errors are visible as a list rather than only as a count. +type Failure struct { + Kind string // "page" or "asset" + URL string + Referer string // the page that referenced it, when known + Reason string // e.g. "HTTP 403 Forbidden" +} + +func (s *stats) recordFailure(f Failure) { + s.muFail.Lock() + if len(s.failures) < maxRecordedFailures { + s.failures = append(s.failures, f) + } + s.muFail.Unlock() +} + +func (s *stats) recordedFailures() []Failure { + s.muFail.Lock() + defer s.muFail.Unlock() + out := make([]Failure, len(s.failures)) + copy(out, s.failures) + return out +} + +// Progress is a snapshot of a run for display. Pages is every page document +// written (it equals the count of HTML files on disk); PagePaths is how many +// distinct URL paths those represent once query strings are ignored. The +// difference, Pages-PagePaths, is the number of query-string variants. +type Progress struct { + Pages int64 + PagePaths int64 + PagesLinked int64 + Assets int64 + AssetSkipped int64 + PageErrors int64 + AssetErrors int64 + Skipped int64 +} + +func (s *stats) snapshot() Progress { + return Progress{ + Pages: s.pages.Load(), + PagePaths: s.pagePaths.Load(), + PagesLinked: s.pagesLinked.Load(), + Assets: s.assets.Load(), + AssetSkipped: s.assetSkipped.Load(), + PageErrors: s.pageErrors.Load(), + AssetErrors: s.assetErrors.Load(), + Skipped: s.skipped.Load(), + } +} + +// Result is the final outcome returned by Run. +type Result struct { + Progress + OutDir string + // Failures is a capped sample of what went wrong, for the final report. + Failures []Failure +} diff --git a/clone/wantasset_test.go b/clone/wantasset_test.go new file mode 100644 index 0000000..fc91d37 --- /dev/null +++ b/clone/wantasset_test.go @@ -0,0 +1,70 @@ +package clone + +import ( + "testing" + + "github.com/tamnd/kage/urlx" +) + +// TestWantAsset checks the two URL-only skip rules a page worker applies before +// downloading an asset: off-domain hosts and bulk-media extensions are left on +// the live web, while the seed's own images, fonts, and stylesheets localize. +func TestWantAsset(t *testing.T) { + seed, _ := urlx.ParseSeed("https://developer.apple.com/") + c := New(seed, DefaultConfig(), nil) + + cases := []struct { + u string + want bool + }{ + // Same registrable domain: localize. + {"https://developer.apple.com/css/main.css", true}, + {"https://www.apple.com/img/logo.png", true}, + {"https://images.apple.com/fonts/sf.woff2", true}, + // Bulk media, installers, archives, and PDFs: leave remote. + {"https://developer.apple.com/videos/wwdc.mp4", false}, + {"https://developer.apple.com/downloads/Xcode.dmg", false}, + {"https://developer.apple.com/bundle.zip", false}, + {"https://developer.apple.com/guide.pdf", false}, + {"https://developer.apple.com/clip.MP4", false}, // case-insensitive + // Off-domain hosts: leave remote even for a normal image. + {"https://cdn-apple.com/img/x.png", false}, + {"https://ec.europa.eu/banner.png", false}, + {"https://mmbiz.qpic.cn/x.jpg", false}, + } + for _, tc := range cases { + u := mustURL(t, tc.u) + if got := c.wantAsset(u); got != tc.want { + t.Errorf("wantAsset(%q) = %v, want %v", tc.u, got, tc.want) + } + } +} + +// TestWantAssetAllHosts checks that turning the domain scope off localizes a +// third-party asset, while the media extension rule still applies. +func TestWantAssetAllHosts(t *testing.T) { + seed, _ := urlx.ParseSeed("https://developer.apple.com/") + cfg := DefaultConfig() + cfg.AssetSameDomain = false + c := New(seed, cfg, nil) + + if !c.wantAsset(mustURL(t, "https://cdn-apple.com/img/x.png")) { + t.Error("with AssetSameDomain off, an off-domain image should localize") + } + if c.wantAsset(mustURL(t, "https://cdn-apple.com/video/x.mp4")) { + t.Error("a media file should still be skipped regardless of host scope") + } +} + +// TestWantAssetKeepMedia checks that an empty skip set (the --keep-media case) +// localizes media too. +func TestWantAssetKeepMedia(t *testing.T) { + seed, _ := urlx.ParseSeed("https://developer.apple.com/") + cfg := DefaultConfig() + cfg.SkipAssetExts = map[string]bool{} + c := New(seed, cfg, nil) + + if !c.wantAsset(mustURL(t, "https://developer.apple.com/videos/wwdc.mp4")) { + t.Error("with an empty skip set, an on-domain video should localize") + } +} diff --git a/cmd/kage/main.go b/cmd/kage/main.go new file mode 100644 index 0000000..51f0cd4 --- /dev/null +++ b/cmd/kage/main.go @@ -0,0 +1,31 @@ +// Command kage clones a website into a self-contained offline folder: it renders +// every page in headless Chrome, strips all JavaScript, and localises the CSS, +// images, and fonts so the saved copy looks like the live site but runs no code. +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + + "github.com/tamnd/kage/cli" + "github.com/tamnd/kage/viewer" +) + +func main() { + // Pin the main goroutine to the process's initial OS thread before anything + // else. In the webview build the native window must be driven from that + // thread; in the default build this is a harmless no-op. + viewer.LockMainThread() + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + go func() { + <-ctx.Done() + stop() + }() + + os.Exit(cli.Execute(ctx)) +} diff --git a/dataset/dataset.go b/dataset/dataset.go new file mode 100644 index 0000000..b412b77 --- /dev/null +++ b/dataset/dataset.go @@ -0,0 +1,307 @@ +// Package dataset converts between a packed ZIM archive and a columnar Parquet +// file. The Parquet form is a flat table with one row per archive entry and +// clear columns (url, mime, title, content, extracted text, redirect target), +// which is the shape a dataset host such as Hugging Face expects. The conversion +// is lossless: every entry, its metadata, and the main page survive a ZIM -> +// Parquet -> ZIM round trip, so the table doubles as an archival representation, +// not just an export. +package dataset + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/google/uuid" + "github.com/parquet-go/parquet-go" + "golang.org/x/net/html" + + "github.com/tamnd/kage/zim" +) + +// Row is one archive entry as a Parquet record. A content or metadata entry +// carries its bytes in Content, its type in Mime, and, for HTML, its visible +// text in Text. A redirect sets IsRedirect and names its destination in +// RedirectTarget ("<namespace>/<url>", e.g. "C/index.html"), with Content empty. +// +// The leading columns (doc_id, url, host, crawl_date, the length pair, text) +// follow the field names the open-index/open-markdown dataset uses, so a kage +// export drops into the same tooling other web-crawl datasets on Hugging Face +// are read with. The trailing columns (namespace, the redirect pair, content) +// are kage's own: they carry the raw bytes and the ZIM structure that make the +// table a lossless, reversible copy of the archive rather than a one-way export. +// +// Namespace is the single-letter ZIM namespace: "C" for pages and assets, "M" +// for archive metadata (Title, Description, Language, ...), "W" for well-known +// entries such as the main-page pointer. Keeping every namespace as a row is +// what makes the table reversible. +type Row struct { + DocID string `parquet:"doc_id,dict"` + URL string `parquet:"url"` + Host string `parquet:"host,dict"` + Title string `parquet:"title"` + Mime string `parquet:"mime,dict"` + CrawlDate string `parquet:"crawl_date,dict"` + ContentLength int64 `parquet:"content_length"` + TextLength int64 `parquet:"text_length"` + Text string `parquet:"text"` + Namespace string `parquet:"namespace,dict"` + IsRedirect bool `parquet:"is_redirect"` + RedirectTarget string `parquet:"redirect_target"` + Content []byte `parquet:"content"` +} + +// docNamespace is the UUID v5 namespace for kage doc_id values: the standard +// URL namespace, matching how open-markdown derives a deterministic id from a +// page's canonical URL. +var docNamespace = uuid.NameSpaceURL + +// docID returns the deterministic UUID v5 an entry gets in the doc_id column, +// derived from its host and url so the same page always hashes to the same id +// across exports. Entries with no host (metadata, well-known) fall back to the +// namespaced url, which is still stable. +func docID(host, namespace, url string) string { + name := namespace + "/" + url + if host != "" { + name = host + "/" + url + } + return uuid.NewSHA1(docNamespace, []byte(name)).String() +} + +// Stats summarises a conversion for the CLI to report. +type Stats struct { + Rows int64 // total entries written + Redirects int64 // of those, redirects + ContentBytes int64 // sum of stored content bytes (uncompressed) +} + +// writeBatch bounds how many rows are buffered before a write to the Parquet +// writer. It only paces the calls; the writer manages its own row groups. +const writeBatch = 256 + +// ZIMToParquet reads the ZIM at zimPath and writes a Parquet table to outPath, +// one row per entry. The archive's main page is recorded both as its W/mainPage +// redirect row and as file-level metadata, and a short generator/source line is +// attached so the dataset is self-describing. version is kage's version string. +func ZIMToParquet(zimPath, outPath, version string) (Stats, error) { + r, err := zim.Open(zimPath) + if err != nil { + return Stats{}, err + } + defer func() { _ = r.Close() }() + + f, err := os.Create(outPath) + if err != nil { + return Stats{}, err + } + bw := bufio.NewWriter(f) + pw := parquet.NewGenericWriter[Row](bw, parquet.Compression(&parquet.Zstd)) + + host := metaValue(r, "Source") + if host == "" { + host = metaValue(r, "Name") + } + host = strings.ToLower(host) + crawlDate := metaValue(r, "Date") + + if ns, u, ok := r.MainPageRef(); ok { + pw.SetKeyValueMetadata("kage.main_page", string(ns)+"/"+u) + } + pw.SetKeyValueMetadata("kage.generator", strings.TrimSpace("kage "+version)) + pw.SetKeyValueMetadata("kage.source", filepath.Base(zimPath)) + if host != "" { + pw.SetKeyValueMetadata("kage.host", host) + } + + var st Stats + count := r.Count() + batch := make([]Row, 0, writeBatch) + flush := func() error { + if len(batch) == 0 { + return nil + } + if _, err := pw.Write(batch); err != nil { + return err + } + batch = batch[:0] + return nil + } + + for i := uint32(0); i < count; i++ { + e, err := r.EntryAt(i) + if err != nil { + return st, fmt.Errorf("read entry %d: %w", i, err) + } + row := Row{ + Namespace: string(e.Namespace), + URL: e.URL, + Title: e.Title, + Host: host, + CrawlDate: crawlDate, + DocID: docID(host, string(e.Namespace), e.URL), + } + if e.Redirect { + row.IsRedirect = true + row.RedirectTarget = string(e.RedirectNamespace) + "/" + e.RedirectURL + st.Redirects++ + } else { + row.Mime = e.MimeType + row.Content = e.Data + row.ContentLength = int64(len(e.Data)) + st.ContentBytes += int64(len(e.Data)) + if e.MimeType == "text/html" { + row.Text = htmlText(e.Data) + row.TextLength = int64(len(row.Text)) + } + } + st.Rows++ + batch = append(batch, row) + if len(batch) == cap(batch) { + if err := flush(); err != nil { + return st, err + } + } + } + if err := flush(); err != nil { + return st, err + } + if err := pw.Close(); err != nil { + return st, err + } + if err := bw.Flush(); err != nil { + return st, err + } + return st, f.Close() +} + +// ParquetToZIM reads the Parquet table at parquetPath and writes the ZIM archive +// it describes to outPath, reproducing every entry, its metadata, and the main +// page. version is unused for now; it is accepted so the signature matches its +// sibling and can record provenance later. +func ParquetToZIM(parquetPath, outPath, _ string) (Stats, error) { + f, err := os.Open(parquetPath) + if err != nil { + return Stats{}, err + } + defer func() { _ = f.Close() }() + + pr := parquet.NewGenericReader[Row](f) + defer func() { _ = pr.Close() }() + + w := zim.NewWriter() + var st Stats + var mainNS byte + var mainURL string + haveMain := false + + buf := make([]Row, writeBatch) + for { + n, readErr := pr.Read(buf) + for i := 0; i < n; i++ { + row := buf[i] + ns := namespaceByte(row.Namespace) + if row.IsRedirect { + tns, turl := splitTarget(row.RedirectTarget) + w.AddRedirect(ns, row.URL, row.Title, tns, turl) + if ns == zim.NamespaceWellKnown && row.URL == "mainPage" { + mainNS, mainURL, haveMain = tns, turl, true + } + st.Redirects++ + } else { + w.AddContent(ns, row.URL, row.Title, row.Mime, row.Content) + st.ContentBytes += int64(len(row.Content)) + } + st.Rows++ + } + if readErr == io.EOF { + break + } + if readErr != nil { + return st, readErr + } + if n == 0 { + break + } + } + if haveMain { + w.SetMainPage(mainNS, mainURL) + } + + out, err := os.Create(outPath) + if err != nil { + return st, err + } + obw := bufio.NewWriter(out) + if _, err := w.WriteTo(obw); err != nil { + _ = out.Close() + return st, err + } + if err := obw.Flush(); err != nil { + _ = out.Close() + return st, err + } + return st, out.Close() +} + +// metaValue reads an M/ metadata entry as a string, returning "" when it is +// absent. It lets the exporter fill the host and crawl_date columns from the +// archive's own metadata. +func metaValue(r *zim.Reader, name string) string { + b, err := r.Get(zim.NamespaceMetadata, name) + if err != nil { + return "" + } + return string(b.Data) +} + +// namespaceByte turns a one-letter namespace column back into the ZIM byte, +// defaulting to the content namespace for an unexpectedly empty value. +func namespaceByte(s string) byte { + if s == "" { + return zim.NamespaceContent + } + return s[0] +} + +// splitTarget parses a "<namespace>/<url>" redirect target back into its parts. +// The namespace is a single byte, so the url is everything after the first two +// characters; a malformed value yields the content namespace and the raw string. +func splitTarget(s string) (byte, string) { + if len(s) >= 2 && s[1] == '/' { + return s[0], s[2:] + } + return zim.NamespaceContent, s +} + +// htmlText extracts the visible text of an HTML document: the concatenated text +// nodes outside script, style, and noscript, with runs of whitespace collapsed +// to single spaces. It is a derived convenience column for dataset consumers and +// plays no part in the round trip, which reconstructs pages from Content. +func htmlText(data []byte) string { + doc, err := html.Parse(strings.NewReader(string(data))) + if err != nil { + return "" + } + var b strings.Builder + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode { + switch n.Data { + case "script", "style", "noscript": + return + } + } + if n.Type == html.TextNode { + b.WriteString(n.Data) + b.WriteByte(' ') + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + return strings.Join(strings.Fields(b.String()), " ") +} diff --git a/dataset/dataset_test.go b/dataset/dataset_test.go new file mode 100644 index 0000000..70d7e03 --- /dev/null +++ b/dataset/dataset_test.go @@ -0,0 +1,115 @@ +package dataset + +import ( + "os" + "path/filepath" + "testing" + + "github.com/tamnd/kage/zim" +) + +// buildZIM writes a small archive with a page, an asset, metadata, and a +// main-page redirect to a temp file, and returns its path. +func buildZIM(t *testing.T) string { + t.Helper() + w := zim.NewWriter() + w.AddContent(zim.NamespaceContent, "index.html", "Home", + "text/html", []byte("<html><head><title>Home

Hello

World

")) + w.AddContent(zim.NamespaceContent, "logo.png", "", "image/png", []byte{0x89, 'P', 'N', 'G', 1, 2, 3}) + w.AddMetadata("Title", "Test Site") + w.AddMetadata("Language", "eng") + w.SetMainPage(zim.NamespaceContent, "index.html") + w.AddRedirect(zim.NamespaceWellKnown, "mainPage", "", zim.NamespaceContent, "index.html") + + path := filepath.Join(t.TempDir(), "site.zim") + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + if _, err := w.WriteTo(f); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + return path +} + +func TestZIMParquetRoundTrip(t *testing.T) { + src := buildZIM(t) + dir := t.TempDir() + pq := filepath.Join(dir, "site.parquet") + dst := filepath.Join(dir, "site2.zim") + + exp, err := ZIMToParquet(src, pq, "test") + if err != nil { + t.Fatalf("ZIMToParquet: %v", err) + } + if exp.Rows == 0 { + t.Fatal("exported zero rows") + } + if exp.Redirects == 0 { + t.Fatal("expected the mainPage redirect to be exported") + } + + imp, err := ParquetToZIM(pq, dst, "test") + if err != nil { + t.Fatalf("ParquetToZIM: %v", err) + } + if imp.Rows != exp.Rows { + t.Fatalf("row count changed across round trip: exported %d, imported %d", exp.Rows, imp.Rows) + } + if imp.Redirects != exp.Redirects { + t.Fatalf("redirect count changed: exported %d, imported %d", exp.Redirects, imp.Redirects) + } + + // The rebuilt archive must serve the same content and resolve its main page. + r, err := zim.Open(dst) + if err != nil { + t.Fatalf("open rebuilt zim: %v", err) + } + defer func() { _ = r.Close() }() + + home, err := r.Get(zim.NamespaceContent, "index.html") + if err != nil { + t.Fatalf("get index.html: %v", err) + } + if got := string(home.Data); got != "Home

Hello

World

" { + t.Fatalf("page content changed: %q", got) + } + if home.MimeType != "text/html" { + t.Fatalf("page mime changed: %q", home.MimeType) + } + + logo, err := r.Get(zim.NamespaceContent, "logo.png") + if err != nil { + t.Fatalf("get logo.png: %v", err) + } + if string(logo.Data) != string([]byte{0x89, 'P', 'N', 'G', 1, 2, 3}) { + t.Fatal("asset bytes changed across round trip") + } + + title, err := r.Get(zim.NamespaceMetadata, "Title") + if err != nil { + t.Fatalf("get M/Title: %v", err) + } + if string(title.Data) != "Test Site" { + t.Fatalf("metadata changed: %q", string(title.Data)) + } + + main, err := r.MainPage() + if err != nil { + t.Fatalf("main page not set after round trip: %v", err) + } + if main.URL != "index.html" { + t.Fatalf("main page url changed: %q", main.URL) + } +} + +func TestHTMLTextStripsScript(t *testing.T) { + got := htmlText([]byte("

Visible

Text\nhere

")) + want := "Visible Text here" + if got != want { + t.Fatalf("htmlText = %q, want %q", got, want) + } +} diff --git a/docs/content/_index.md b/docs/content/_index.md new file mode 100644 index 0000000..139ec7c --- /dev/null +++ b/docs/content/_index.md @@ -0,0 +1,38 @@ +--- +title: "kage" +description: "kage (影, shadow) clones any website into a self-contained folder you can browse offline, with all the JavaScript stripped out. Render in headless Chrome, remove every script, localise the CSS, images, and fonts, from one pure-Go binary." +heroTitle: "A website, frozen as a shadow" +heroLead: "kage renders every page in headless Chrome, snapshots the final DOM, removes every script and event handler, and downloads and rewrites the CSS, images, and fonts. The result looks like the live site but runs no code: a plain folder of .html files you can open straight from disk." +heroPrimaryURL: "/getting-started/quick-start/" +heroPrimaryText: "Get started" +--- + +Saving a page with "Save As" gives you a copy that still phones home, still runs analytics, and often renders blank because the markup is built by JavaScript at runtime. kage (影, "shadow") takes the opposite approach: it drives a real browser, captures the page the way a human would have seen it, then makes it inert. + +Say you want Paul Graham's essays on a laptop with no wifi. One command mirrors the site; a second serves it back offline: + +```bash +kage clone paulgraham.com +kage serve $HOME/data/kage/paulgraham.com +``` + +![kage cloning paulgraham.com, packing it into one file, and serving it back offline](/demo.gif) + +## What it does + +- **Renders first, saves second.** Each page goes through real headless Chrome, so a page whose content is assembled by JavaScript is captured fully, not as an empty shell. +- **Strips every script.** Once the DOM is captured, kage removes all ` + + + +

Title

+js link +real link + +
+ +

styled

+` + +func TestStripRemovesAllJS(t *testing.T) { + out, rep, err := Strip([]byte(page), Options{Banner: "cloned by kage"}) + if err != nil { + t.Fatal(err) + } + s := string(out) + + if strings.Contains(s, " survived") + } + if strings.Contains(s, "onload") || strings.Contains(s, "onclick") || strings.Contains(s, "onerror") { + t.Error("an on* handler survived") + } + if strings.Contains(strings.ToLower(s), "javascript:") { + t.Error("a javascript: URL survived") + } + if strings.Contains(s, "modulepreload") || strings.Contains(s, "preconnect") { + t.Error("a dead resource hint survived") + } + if strings.Contains(s, "http-equiv") { + t.Error("a meta refresh survived") + } + + // Layout-bearing markup must survive untouched. + for _, want := range []string{ + `rel="stylesheet"`, `href="/css/main.css"`, + `