Add the clone engine, CLI, tests, CI, and docs
kage renders every page in headless Chrome, snapshots the final DOM, strips all JavaScript, and localises CSS, images, and fonts so a site can be browsed offline as a plain folder of files. The engine is split into small packages: urlx deterministic URL to local-path mapping and scope rules sanitize remove scripts, on* handlers, and javascript: URLs asset rewrite HTML and CSS references, download assets browser headless Chrome pool over the DevTools protocol robots robots.txt matcher clone the orchestrator: a polite resumable breadth-first crawl The cli package wires a cobra and fang command surface with two commands, clone and serve. Every pure package has table tests; the browser and clone packages add Chrome-driven end-to-end tests that skip when no browser is present or under -short. CI runs gofmt, vet, build, race tests, golangci-lint, govulncheck, and a tidy check on Linux and macOS. A goreleaser config fans one tag out to archives, deb/rpm/apk, a Chromium-bundled GHCR image, and the package managers. A tago docs site builds to Pages and Cloudflare.
This commit is contained in:
Duc-Tam Nguyen
@@ -0,0 +1,103 @@
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Cancel an in-flight run when a branch is pushed again.
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Build and test on Linux and macOS with the race detector on. gofmt and vet
|
||||
# run here too so a formatting slip fails fast on both platforms. The full
|
||||
# suite includes Chrome-driven end-to-end tests, so the runners install a
|
||||
# browser first; tests that find none skip themselves, so the job still passes
|
||||
# if a future runner image drops Chrome.
|
||||
test:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: true
|
||||
- name: install chromium (linux)
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
uses: browser-actions/setup-chrome@v1
|
||||
id: chrome
|
||||
- name: gofmt
|
||||
run: |
|
||||
unformatted=$(gofmt -s -l asset browser cli clone cmd robots sanitize urlx)
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "These files need gofmt -s -w:"
|
||||
echo "$unformatted"
|
||||
exit 1
|
||||
fi
|
||||
- name: go vet
|
||||
run: go vet ./...
|
||||
- name: build
|
||||
run: go build ./...
|
||||
- name: test
|
||||
env:
|
||||
KAGE_CHROME: ${{ steps.chrome.outputs.chrome-path }}
|
||||
run: go test -race -count=1 -coverprofile=coverage.out ./...
|
||||
- name: coverage summary
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
run: go tool cover -func=coverage.out | tail -1
|
||||
|
||||
# golangci-lint bundles staticcheck, govet, ineffassign, errcheck, unused and
|
||||
# more, so it is the main quality gate.
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: true
|
||||
- uses: golangci/golangci-lint-action@v8
|
||||
with:
|
||||
version: latest
|
||||
|
||||
# Scan the module and its dependencies for known vulnerabilities.
|
||||
govulncheck:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: true
|
||||
- name: govulncheck
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
govulncheck ./...
|
||||
|
||||
# Confirm go.mod and go.sum are tidy: a PR that adds an import without running
|
||||
# go mod tidy fails here instead of breaking a later release.
|
||||
tidy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: true
|
||||
- name: go mod tidy is clean
|
||||
run: |
|
||||
go mod tidy
|
||||
git diff --exit-code -- go.mod go.sum
|
||||
@@ -0,0 +1,135 @@
|
||||
name: Docs
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "docs/**"
|
||||
- ".github/workflows/docs.yml"
|
||||
- "scripts/ensure_cloudflare_pages.py"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
deployments: write
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
|
||||
concurrency:
|
||||
group: docs-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6.0.2
|
||||
with:
|
||||
submodules: true
|
||||
# Sitemap lastmod comes from the latest content commit.
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Checkout tago
|
||||
uses: actions/checkout@v6.0.2
|
||||
with:
|
||||
repository: tamnd/tago
|
||||
path: .tago-src
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v6.4.0
|
||||
with:
|
||||
go-version-file: .tago-src/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Cache tago binary
|
||||
id: tago-bin-cache
|
||||
uses: actions/cache@v5.0.5
|
||||
with:
|
||||
path: /usr/local/bin/tago
|
||||
key: tago-bin-${{ runner.os }}-${{ hashFiles('.tago-src/go.sum', '.tago-src/**/*.go') }}
|
||||
|
||||
- name: Build tago
|
||||
if: steps.tago-bin-cache.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
cd .tago-src
|
||||
go build -o /usr/local/bin/tago ./cmd/tago/
|
||||
|
||||
# Two builds, one per deploy target. The theme links every page and asset
|
||||
# with absURL, so each output carries its own base path: the Pages build
|
||||
# gets the /kage/ sub-path, the Cloudflare build gets the domain root.
|
||||
- name: Build for GitHub Pages
|
||||
working-directory: docs
|
||||
run: tago build --base-url "https://tamnd.github.io/kage/" --output public-pages
|
||||
|
||||
# tago absURLs theme links but passes markdown content links through, so
|
||||
# root-relative links in content need the sub-path prefixed for the Pages
|
||||
# mirror. The Cloudflare build serves from the root and needs none.
|
||||
- name: Rewrite content links for the Pages sub-path
|
||||
run: |
|
||||
find docs/public-pages -name '*.html' -print0 |
|
||||
xargs -0 perl -pi -e 's|(href=")/(?!/)|${1}/kage/|g; s|(src=")/(?!/)|${1}/kage/|g'
|
||||
|
||||
- name: Upload Pages artifact
|
||||
uses: actions/upload-pages-artifact@v5.0.0
|
||||
with:
|
||||
path: docs/public-pages
|
||||
|
||||
- name: Build for Cloudflare Pages
|
||||
working-directory: docs
|
||||
run: tago build --base-url "https://kage.tamnd.com/" --output public-cf
|
||||
|
||||
- name: Upload Cloudflare artifact
|
||||
uses: actions/upload-artifact@v7.0.1
|
||||
with:
|
||||
name: public-cf
|
||||
path: docs/public-cf
|
||||
retention-days: 1
|
||||
|
||||
deploy-pages:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v5.0.0
|
||||
|
||||
deploy-cloudflare:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
concurrency:
|
||||
group: cloudflare-pages-kage
|
||||
cancel-in-progress: true
|
||||
steps:
|
||||
- uses: actions/checkout@v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
sparse-checkout: scripts/
|
||||
|
||||
- name: Download Cloudflare artifact
|
||||
uses: actions/download-artifact@v8.0.1
|
||||
with:
|
||||
name: public-cf
|
||||
path: public-cf
|
||||
|
||||
- name: Ensure Cloudflare Pages config
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
||||
run: python3 scripts/ensure_cloudflare_pages.py --project kage --domain kage.tamnd.com --zone tamnd.com
|
||||
|
||||
- name: Deploy to Cloudflare Pages
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
||||
run: |
|
||||
npx -y wrangler@4 pages deploy public-cf \
|
||||
--project-name=kage \
|
||||
--branch=main \
|
||||
--commit-dirty=true
|
||||
@@ -0,0 +1,87 @@
|
||||
name: release
|
||||
|
||||
# Pushing a version tag turns the GoReleaser config (.goreleaser.yaml) into a
|
||||
# GitHub release with the archives, Linux packages (deb, rpm, apk), checksums,
|
||||
# SBOMs and a cosign signature, and pushes the multi-arch container image to
|
||||
# GHCR. Pull requests and pushes to main run `goreleaser check` only, so a
|
||||
# config that would fail a real release is caught long before the tag.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Fast gate on every PR and push: the config parses and is valid for the
|
||||
# installed GoReleaser version. No artifacts are built here.
|
||||
check:
|
||||
if: github.ref_type != 'tag'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: true
|
||||
- uses: goreleaser/goreleaser-action@v6
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: "~> v2"
|
||||
args: check
|
||||
|
||||
# The real release, only on a version tag.
|
||||
release:
|
||||
if: github.ref_type == 'tag'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write # create the GitHub release
|
||||
packages: write # push the image to ghcr.io
|
||||
id-token: write # keyless cosign signing
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: true
|
||||
|
||||
# Build and ship the linux/arm64 image from the amd64 runner.
|
||||
- uses: docker/setup-qemu-action@v3
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Tools GoReleaser shells out to for signing and SBOMs.
|
||||
- uses: sigstore/cosign-installer@v3
|
||||
- uses: anchore/sbom-action/download-syft@v0
|
||||
|
||||
- uses: goreleaser/goreleaser-action@v6
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: "~> v2"
|
||||
args: release --clean
|
||||
env:
|
||||
# Creating the release and pushing the GHCR image use the built-in
|
||||
# token. The package-manager publish steps each read their own secret;
|
||||
# any that is unset leaves that manager skipped (the artifact is still
|
||||
# produced), so the release never fails for a tap that is not set up.
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }}
|
||||
SCOOP_BUCKET_GITHUB_TOKEN: ${{ secrets.SCOOP_BUCKET_GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,192 @@
|
||||
# GoReleaser turns one tag push into everything a user might install from: raw
|
||||
# archives, Linux packages (deb, rpm, apk), a multi-arch container image, and
|
||||
# entries for the package managers (Homebrew, Scoop). `git tag vX.Y.Z && git
|
||||
# push --tags` fans out to all of them through .github/workflows/release.yml.
|
||||
#
|
||||
# Publish steps that push to a repository we do not own yet (the Homebrew tap,
|
||||
# the Scoop bucket) self-disable when their token is absent. A release with no
|
||||
# extra secrets still produces every downloadable artifact and the container
|
||||
# image; each manager lights up the moment its repository and token exist.
|
||||
version: 2
|
||||
|
||||
project_name: kage
|
||||
|
||||
before:
|
||||
# Only fetch modules; never `go mod tidy` during a release, the tree is kept
|
||||
# tidy in CI.
|
||||
hooks:
|
||||
- go mod download
|
||||
|
||||
builds:
|
||||
- id: kage
|
||||
binary: kage
|
||||
main: ./cmd/kage
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
flags:
|
||||
- -trimpath
|
||||
ldflags:
|
||||
- -s -w
|
||||
- -X github.com/tamnd/kage/cli.Version={{ .Version }}
|
||||
- -X github.com/tamnd/kage/cli.Commit={{ .ShortCommit }}
|
||||
- -X github.com/tamnd/kage/cli.Date={{ .CommitDate }}
|
||||
mod_timestamp: "{{ .CommitTimestamp }}"
|
||||
targets:
|
||||
- linux_amd64
|
||||
- linux_arm64
|
||||
- linux_arm_7
|
||||
- linux_386
|
||||
- darwin_amd64
|
||||
- darwin_arm64
|
||||
- windows_amd64
|
||||
- windows_arm64
|
||||
- freebsd_amd64
|
||||
- freebsd_arm64
|
||||
|
||||
archives:
|
||||
# tar.gz everywhere except a zip on Windows.
|
||||
- id: default
|
||||
name_template: "kage_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}"
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
formats: [zip]
|
||||
files:
|
||||
- LICENSE
|
||||
- README.md
|
||||
|
||||
nfpms:
|
||||
# One nfpm definition emits the deb, rpm, and apk for every Linux build. kage
|
||||
# is a user command, not a daemon, so there is no unit file and no
|
||||
# postinstall: the package is the binary and its license. Chrome is a runtime
|
||||
# dependency the user supplies (or the container image bundles).
|
||||
- id: linux-packages
|
||||
package_name: kage
|
||||
file_name_template: "{{ .ConventionalFileName }}"
|
||||
vendor: tamnd
|
||||
homepage: https://github.com/tamnd/kage
|
||||
maintainer: Duc-Tam Nguyen <tamnd87@gmail.com>
|
||||
description: Clone any website for offline viewing, with the JavaScript stripped out.
|
||||
license: MIT
|
||||
formats:
|
||||
- deb
|
||||
- rpm
|
||||
- apk
|
||||
bindir: /usr/bin
|
||||
section: utils
|
||||
recommends:
|
||||
- chromium
|
||||
contents:
|
||||
- src: ./LICENSE
|
||||
dst: /usr/share/doc/kage/LICENSE
|
||||
|
||||
dockers_v2:
|
||||
# One multi-platform image built with buildx. GoReleaser stages the prebuilt
|
||||
# binaries under per-platform directories in the build context and the
|
||||
# Dockerfile copies the right one through $TARGETPLATFORM.
|
||||
- images:
|
||||
- ghcr.io/tamnd/kage
|
||||
tags:
|
||||
- "{{ .Version }}"
|
||||
- latest
|
||||
dockerfile: Dockerfile
|
||||
platforms:
|
||||
- linux/amd64
|
||||
- linux/arm64
|
||||
labels:
|
||||
org.opencontainers.image.title: "{{ .ProjectName }}"
|
||||
org.opencontainers.image.description: "Clone any website for offline viewing, with the JavaScript stripped out"
|
||||
org.opencontainers.image.url: "https://github.com/tamnd/kage"
|
||||
org.opencontainers.image.source: "https://github.com/tamnd/kage"
|
||||
org.opencontainers.image.version: "{{ .Version }}"
|
||||
org.opencontainers.image.revision: "{{ .FullCommit }}"
|
||||
org.opencontainers.image.licenses: "MIT"
|
||||
|
||||
homebrew_casks:
|
||||
# Homebrew cask pushed to the tap repository. Self-disables until
|
||||
# HOMEBREW_TAP_GITHUB_TOKEN (a PAT with write to tamnd/homebrew-tap) is set,
|
||||
# so a tokenless release still writes the cask into dist for inspection.
|
||||
- name: kage
|
||||
repository:
|
||||
owner: tamnd
|
||||
name: homebrew-tap
|
||||
token: '{{ envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}'
|
||||
directory: Casks
|
||||
homepage: https://github.com/tamnd/kage
|
||||
description: Clone any website for offline viewing, with the JavaScript stripped out
|
||||
skip_upload: '{{ if envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}'
|
||||
commit_author:
|
||||
name: Duc-Tam Nguyen
|
||||
email: tamnd87@gmail.com
|
||||
|
||||
scoops:
|
||||
# Scoop manifest for Windows, pushed to the bucket repository.
|
||||
- repository:
|
||||
owner: tamnd
|
||||
name: scoop-bucket
|
||||
token: '{{ envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}'
|
||||
homepage: https://github.com/tamnd/kage
|
||||
description: Clone any website for offline viewing, with the JavaScript stripped out
|
||||
license: MIT
|
||||
skip_upload: '{{ if envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}'
|
||||
commit_author:
|
||||
name: Duc-Tam Nguyen
|
||||
email: tamnd87@gmail.com
|
||||
|
||||
checksum:
|
||||
name_template: "checksums.txt"
|
||||
algorithm: sha256
|
||||
|
||||
sboms:
|
||||
# A CycloneDX SBOM per archive via syft; the release workflow installs it.
|
||||
- id: archive
|
||||
artifacts: archive
|
||||
|
||||
signs:
|
||||
# Keyless cosign signature over the checksum file. It runs only on a real
|
||||
# release in CI, where the workflow grants the OIDC token cosign needs.
|
||||
- cmd: cosign
|
||||
certificate: "${artifact}.pem"
|
||||
args:
|
||||
- sign-blob
|
||||
- "--output-certificate=${certificate}"
|
||||
- "--output-signature=${signature}"
|
||||
- "${artifact}"
|
||||
- "--yes"
|
||||
artifacts: checksum
|
||||
output: true
|
||||
|
||||
docker_signs:
|
||||
- cmd: cosign
|
||||
artifacts: manifests
|
||||
args:
|
||||
- sign
|
||||
- "${artifact}@${digest}"
|
||||
- "--yes"
|
||||
|
||||
changelog:
|
||||
sort: asc
|
||||
use: github
|
||||
filters:
|
||||
exclude:
|
||||
- "^docs:"
|
||||
- "^test:"
|
||||
- "^chore:"
|
||||
- "^ci:"
|
||||
- Merge pull request
|
||||
- Merge branch
|
||||
groups:
|
||||
- title: Features
|
||||
regexp: '^.*?feat(\(.+\))??!?:.+$'
|
||||
order: 0
|
||||
- title: Fixes
|
||||
regexp: '^.*?fix(\(.+\))??!?:.+$'
|
||||
order: 1
|
||||
- title: Other
|
||||
order: 999
|
||||
|
||||
release:
|
||||
github:
|
||||
owner: tamnd
|
||||
name: kage
|
||||
draft: false
|
||||
prerelease: auto
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
# Consumed by GoReleaser: it copies the already cross-compiled binary out of the
|
||||
# build context rather than compiling, so the image build is fast and uses the
|
||||
# same static binary every other artifact ships.
|
||||
#
|
||||
# kage always drives a real headless Chrome, so unlike a plain CLI image this one
|
||||
# bundles Chromium. KAGE_CHROME points kage at the system binary so it never
|
||||
# tries to download its own.
|
||||
#
|
||||
# GoReleaser builds one multi-platform image with buildx and stages each
|
||||
# platform's binary under a $TARGETPLATFORM directory (e.g. linux/amd64/) in the
|
||||
# build context, so the COPY line selects the right one through the automatic
|
||||
# TARGETPLATFORM build arg.
|
||||
FROM alpine:3.21
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
|
||||
# chromium for rendering; ca-certificates for HTTPS; tzdata for sane timestamps;
|
||||
# the font package so rendered pages have glyphs to lay out.
|
||||
RUN apk add --no-cache chromium ca-certificates tzdata font-noto \
|
||||
&& adduser -D -H -u 10001 kage \
|
||||
&& mkdir -p /out \
|
||||
&& chown kage:kage /out
|
||||
|
||||
COPY $TARGETPLATFORM/kage /usr/bin/kage
|
||||
|
||||
USER kage
|
||||
WORKDIR /out
|
||||
|
||||
# Point kage at the bundled Chromium and write mirrors under /out by default:
|
||||
#
|
||||
# docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com
|
||||
ENV KAGE_CHROME=/usr/bin/chromium-browser
|
||||
|
||||
VOLUME ["/out"]
|
||||
|
||||
ENTRYPOINT ["/usr/bin/kage"]
|
||||
@@ -0,0 +1,39 @@
|
||||
BIN := kage
|
||||
PKG := ./cmd/kage
|
||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
||||
COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo none)
|
||||
DATE := $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
LDFLAGS := -s -w \
|
||||
-X github.com/tamnd/kage/cli.Version=$(VERSION) \
|
||||
-X github.com/tamnd/kage/cli.Commit=$(COMMIT) \
|
||||
-X github.com/tamnd/kage/cli.Date=$(DATE)
|
||||
|
||||
export CGO_ENABLED := 0
|
||||
|
||||
.PHONY: build install test test-short vet tidy clean run
|
||||
|
||||
build:
|
||||
go build -ldflags "$(LDFLAGS)" -o bin/$(BIN) $(PKG)
|
||||
|
||||
install:
|
||||
go install -ldflags "$(LDFLAGS)" $(PKG)
|
||||
|
||||
# Full suite, including the Chrome-driven end-to-end tests.
|
||||
test:
|
||||
go test -race ./...
|
||||
|
||||
# Skip the tests that launch a real browser (CI without Chrome, quick loops).
|
||||
test-short:
|
||||
go test -short ./...
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
tidy:
|
||||
go mod tidy
|
||||
|
||||
clean:
|
||||
rm -rf bin
|
||||
|
||||
run: build
|
||||
./bin/$(BIN)
|
||||
@@ -0,0 +1,116 @@
|
||||
package asset
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/tamnd/kage/urlx"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// testSink builds a RefSink that records every URL it sees and rewrites it to a
|
||||
// local relative path from fromDir, mirroring what the cloner does.
|
||||
func testSink(seedHost, fromDir string, seen *[]string) RefSink {
|
||||
return func(u *url.URL, kind urlx.Kind) string {
|
||||
*seen = append(*seen, u.String())
|
||||
if !urlx.SameSite(&url.URL{Host: seedHost}, u, true) && kind == urlx.Page {
|
||||
return u.String() // external page links stay absolute
|
||||
}
|
||||
local := urlx.LocalPath(seedHost, u, kind, "")
|
||||
return urlx.Rel(fromDir, local)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteCSS(t *testing.T) {
|
||||
base, _ := url.Parse("https://ex.com/css/main.css")
|
||||
in := `@import "base.css";
|
||||
@import url('https://cdn.io/reset.css');
|
||||
body { background: url(../img/bg.png); }
|
||||
.x { background: url("https://cdn.io/sprite.svg"); }
|
||||
.y { content: url(data:image/gif;base64,AAAA); }`
|
||||
var seen []string
|
||||
// CSS lives at _kage/ex.com/css/main.css → its dir.
|
||||
fromDir := "_kage/ex.com/css"
|
||||
out := string(RewriteCSS([]byte(in), base, testSink("ex.com", fromDir, &seen)))
|
||||
|
||||
if !strings.Contains(out, `@import "base.css"`) {
|
||||
t.Errorf("local @import not rewritten relative: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, `url("../img/bg.png")`) {
|
||||
t.Errorf("local url() not rewritten relative: %s", out)
|
||||
}
|
||||
// Cross-origin asset goes under _kage/cdn.io/...; from _kage/ex.com/css that
|
||||
// is ../../cdn.io/...
|
||||
if !strings.Contains(out, `url("../../cdn.io/reset.css")`) {
|
||||
t.Errorf("cross-origin @import not localised: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, `url("../../cdn.io/sprite.svg")`) {
|
||||
t.Errorf("cross-origin url() not localised: %s", out)
|
||||
}
|
||||
// data: URL must be left exactly as-is.
|
||||
if !strings.Contains(out, "url(data:image/gif;base64,AAAA)") {
|
||||
t.Errorf("data: URL was altered: %s", out)
|
||||
}
|
||||
|
||||
sort.Strings(seen)
|
||||
want := []string{
|
||||
"https://cdn.io/reset.css",
|
||||
"https://cdn.io/sprite.svg",
|
||||
"https://ex.com/css/base.css",
|
||||
"https://ex.com/img/bg.png",
|
||||
}
|
||||
if strings.Join(seen, ",") != strings.Join(want, ",") {
|
||||
t.Errorf("sink saw %v, want %v", seen, want)
|
||||
}
|
||||
}
|
||||
|
||||
const htmlDoc = `<!doctype html><html><head>
|
||||
<link rel="stylesheet" href="/css/main.css">
|
||||
<link rel="icon" href="/favicon.ico">
|
||||
<link rel="canonical" href="https://ex.com/canon">
|
||||
</head><body>
|
||||
<a href="/docs/intro">internal</a>
|
||||
<a href="https://other.com/x">external</a>
|
||||
<a href="/files/report.pdf">a pdf</a>
|
||||
<img src="/img/logo.png" srcset="/img/logo.png 1x, /img/logo@2x.png 2x">
|
||||
<p style="background:url(/img/bg.png)">hi</p>
|
||||
<style>.a{background:url(/img/tile.png)}</style>
|
||||
</body></html>`
|
||||
|
||||
func TestRewriteHTML(t *testing.T) {
|
||||
base, _ := url.Parse("https://ex.com/")
|
||||
root, err := html.Parse(strings.NewReader(htmlDoc))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var seen []string
|
||||
// The page is the root index.html, so fromDir is "".
|
||||
RewriteHTML(root, base, testSink("ex.com", "", &seen))
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := html.Render(&buf, root); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := buf.String()
|
||||
|
||||
checks := map[string]bool{
|
||||
`href="_kage/ex.com/css/main.css"`: true, // stylesheet localised
|
||||
`href="_kage/ex.com/favicon.ico"`: true, // icon localised
|
||||
`href="https://ex.com/canon"`: true, // canonical left alone
|
||||
`href="docs/intro/index.html"`: true, // internal page → local
|
||||
`href="https://other.com/x"`: true, // external page stays absolute
|
||||
`href="_kage/ex.com/files/report.pdf"`: true, // pdf link treated as asset
|
||||
`src="_kage/ex.com/img/logo.png"`: true,
|
||||
`_kage/ex.com/img/logo@2x.png 2x`: true, // srcset candidate rewritten
|
||||
`background:url("_kage/ex.com/img/bg.png")`: true, // inline style (attr-escaped quotes)
|
||||
`background:url("_kage/ex.com/img/tile.png")`: true, // <style> text
|
||||
}
|
||||
for frag, want := range checks {
|
||||
if strings.Contains(out, frag) != want {
|
||||
t.Errorf("fragment %q present=%v, want %v\n--- output ---\n%s", frag, !want, want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package asset
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/tamnd/kage/urlx"
|
||||
)
|
||||
|
||||
// RefSink registers a resolved asset/page URL with the cloner and returns the
|
||||
// string to write back into the markup or CSS — a relative local path for
|
||||
// things kage saves, or the absolute URL for anything it leaves on the live web.
|
||||
type RefSink func(u *url.URL, kind urlx.Kind) string
|
||||
|
||||
var (
|
||||
// url( ... ) with optional single/double quotes or bare token.
|
||||
cssURLRe = regexp.MustCompile(`url\(\s*("[^"]*"|'[^']*'|[^)'"]*)\s*\)`)
|
||||
// @import "..." / @import '...' (the @import url(...) form is caught by cssURLRe).
|
||||
cssImportRe = regexp.MustCompile(`@import\s+("[^"]*"|'[^']*')`)
|
||||
)
|
||||
|
||||
// RewriteCSS rewrites every url(...) and @import in a stylesheet so its
|
||||
// references point at local files. base is the stylesheet's own URL (so relative
|
||||
// references resolve correctly); sink maps each absolute URL to its local path.
|
||||
// data: URLs and unparseable references are left untouched.
|
||||
func RewriteCSS(css []byte, base *url.URL, sink RefSink) []byte {
|
||||
s := string(css)
|
||||
s = cssImportRe.ReplaceAllStringFunc(s, func(m string) string {
|
||||
raw := cssImportRe.FindStringSubmatch(m)[1]
|
||||
ref := unquote(raw)
|
||||
if newRef, ok := resolveRef(base, ref, sink); ok {
|
||||
return `@import "` + newRef + `"`
|
||||
}
|
||||
return m
|
||||
})
|
||||
s = cssURLRe.ReplaceAllStringFunc(s, func(m string) string {
|
||||
raw := cssURLRe.FindStringSubmatch(m)[1]
|
||||
ref := unquote(raw)
|
||||
if newRef, ok := resolveRef(base, ref, sink); ok {
|
||||
return `url("` + newRef + `")`
|
||||
}
|
||||
return m
|
||||
})
|
||||
return []byte(s)
|
||||
}
|
||||
|
||||
// resolveRef normalises ref against base and runs it through the sink. It
|
||||
// reports ok=false (leave the original text) for empty, data:, or unparseable
|
||||
// references.
|
||||
func resolveRef(base *url.URL, ref string, sink RefSink) (string, bool) {
|
||||
ref = strings.TrimSpace(ref)
|
||||
if ref == "" || strings.HasPrefix(strings.ToLower(ref), "data:") || strings.HasPrefix(ref, "#") {
|
||||
return "", false
|
||||
}
|
||||
u, err := urlx.Normalize(base, ref)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return sink(u, urlx.Asset), true
|
||||
}
|
||||
|
||||
func unquote(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) >= 2 && (s[0] == '"' || s[0] == '\'') && s[len(s)-1] == s[0] {
|
||||
return s[1 : len(s)-1]
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package asset
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Downloader fetches asset bytes over plain HTTP. It is separate from the Chrome
|
||||
// pool: assets are public bytes that rarely need a real browser, so a fast HTTP
|
||||
// client keeps the crawl cheap. Failures are returned to the caller, which logs
|
||||
// them and moves on — a missing asset degrades a page, it never aborts a clone.
|
||||
type Downloader struct {
|
||||
Client *http.Client
|
||||
UserAgent string
|
||||
MaxBytes int64 // per-asset cap; 0 = unlimited
|
||||
}
|
||||
|
||||
// NewDownloader builds a Downloader with a sane client and the given timeout.
|
||||
func NewDownloader(userAgent string, timeout time.Duration, maxBytes int64) *Downloader {
|
||||
return &Downloader{
|
||||
Client: &http.Client{Timeout: timeout},
|
||||
UserAgent: userAgent,
|
||||
MaxBytes: maxBytes,
|
||||
}
|
||||
}
|
||||
|
||||
// Result is a downloaded asset.
|
||||
type Result struct {
|
||||
Body []byte
|
||||
ContentType string
|
||||
IsCSS bool
|
||||
}
|
||||
|
||||
// Get fetches u, sending referer as the Referer header. It reads at most
|
||||
// MaxBytes and reports whether the body is CSS (so the caller can rewrite it).
|
||||
func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Result, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d.UserAgent != "" {
|
||||
req.Header.Set("User-Agent", d.UserAgent)
|
||||
}
|
||||
if referer != "" {
|
||||
req.Header.Set("Referer", referer)
|
||||
}
|
||||
resp, err := d.Client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("status %d for %s", resp.StatusCode, u)
|
||||
}
|
||||
var r io.Reader = resp.Body
|
||||
if d.MaxBytes > 0 {
|
||||
r = io.LimitReader(resp.Body, d.MaxBytes)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
return &Result{
|
||||
Body: body,
|
||||
ContentType: ct,
|
||||
IsCSS: isCSS(ct, u),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// isCSS reports whether a response is a stylesheet, by content-type or by a
|
||||
// .css path when the server sends no useful type.
|
||||
func isCSS(contentType string, u *url.URL) bool {
|
||||
if strings.Contains(strings.ToLower(contentType), "text/css") {
|
||||
return true
|
||||
}
|
||||
return strings.HasSuffix(strings.ToLower(u.Path), ".css")
|
||||
}
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
package asset
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/tamnd/kage/urlx"
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/net/html/atom"
|
||||
)
|
||||
|
||||
// stylesheetRels are <link rel> values whose href kage downloads as an asset.
|
||||
var stylesheetRels = map[string]bool{
|
||||
"stylesheet": true, "icon": true, "shortcut icon": true,
|
||||
"apple-touch-icon": true, "apple-touch-icon-precomposed": true,
|
||||
"mask-icon": true, "manifest": true, "preload": true, "prefetch": true,
|
||||
}
|
||||
|
||||
// RewriteHTML walks the parsed document and rewrites every resource and link
|
||||
// reference through sink, resolving relative URLs against base. It mutates the
|
||||
// tree in place; the caller renders it afterwards. References kage cannot handle
|
||||
// (data:, mailto:, fragment-only, …) are left untouched.
|
||||
func RewriteHTML(root *html.Node, base *url.URL, sink RefSink) {
|
||||
var walk func(n *html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
rewriteElement(n, base, sink)
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(root)
|
||||
}
|
||||
|
||||
func rewriteElement(n *html.Node, base *url.URL, sink RefSink) {
|
||||
switch n.DataAtom {
|
||||
case atom.A, atom.Area:
|
||||
rewriteAttr(n, "href", base, sink, pageOrAsset)
|
||||
case atom.Iframe, atom.Frame:
|
||||
rewriteAttr(n, "src", base, sink, pageOrAsset)
|
||||
case atom.Link:
|
||||
rel := strings.ToLower(strings.TrimSpace(attrVal(n, "rel")))
|
||||
if stylesheetRels[rel] {
|
||||
rewriteAttr(n, "href", base, sink, alwaysAsset)
|
||||
}
|
||||
case atom.Img:
|
||||
rewriteAttr(n, "src", base, sink, alwaysAsset)
|
||||
rewriteSrcset(n, base, sink)
|
||||
case atom.Source:
|
||||
rewriteAttr(n, "src", base, sink, alwaysAsset)
|
||||
rewriteSrcset(n, base, sink)
|
||||
case atom.Video:
|
||||
rewriteAttr(n, "src", base, sink, alwaysAsset)
|
||||
rewriteAttr(n, "poster", base, sink, alwaysAsset)
|
||||
case atom.Audio, atom.Track, atom.Embed:
|
||||
rewriteAttr(n, "src", base, sink, alwaysAsset)
|
||||
case atom.Object:
|
||||
rewriteAttr(n, "data", base, sink, alwaysAsset)
|
||||
case atom.Style:
|
||||
rewriteStyleText(n, base, sink)
|
||||
}
|
||||
// Any element may carry an inline style="" with url() references.
|
||||
rewriteInlineStyle(n, base, sink)
|
||||
}
|
||||
|
||||
// kindFunc decides whether a normalized URL is a page or an asset.
|
||||
type kindFunc func(u *url.URL) urlx.Kind
|
||||
|
||||
func alwaysAsset(*url.URL) urlx.Kind { return urlx.Asset }
|
||||
|
||||
func pageOrAsset(u *url.URL) urlx.Kind {
|
||||
if urlx.LikelyPage(u) {
|
||||
return urlx.Page
|
||||
}
|
||||
return urlx.Asset
|
||||
}
|
||||
|
||||
func rewriteAttr(n *html.Node, key string, base *url.URL, sink RefSink, kf kindFunc) {
|
||||
for i := range n.Attr {
|
||||
if !strings.EqualFold(n.Attr[i].Key, key) {
|
||||
continue
|
||||
}
|
||||
u, err := urlx.Normalize(base, n.Attr[i].Val)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
n.Attr[i].Val = sink(u, kf(u))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// rewriteSrcset rewrites each candidate URL in a srcset attribute, keeping the
|
||||
// width/density descriptors intact.
|
||||
func rewriteSrcset(n *html.Node, base *url.URL, sink RefSink) {
|
||||
for i := range n.Attr {
|
||||
if !strings.EqualFold(n.Attr[i].Key, "srcset") {
|
||||
continue
|
||||
}
|
||||
n.Attr[i].Val = rewriteSrcsetValue(n.Attr[i].Val, base, sink)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func rewriteSrcsetValue(val string, base *url.URL, sink RefSink) string {
|
||||
parts := strings.Split(val, ",")
|
||||
for i, p := range parts {
|
||||
fields := strings.Fields(strings.TrimSpace(p))
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
u, err := urlx.Normalize(base, fields[0])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
fields[0] = sink(u, urlx.Asset)
|
||||
parts[i] = strings.Join(fields, " ")
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
func rewriteInlineStyle(n *html.Node, base *url.URL, sink RefSink) {
|
||||
for i := range n.Attr {
|
||||
if !strings.EqualFold(n.Attr[i].Key, "style") {
|
||||
continue
|
||||
}
|
||||
n.Attr[i].Val = string(RewriteCSS([]byte(n.Attr[i].Val), base, sink))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func rewriteStyleText(n *html.Node, base *url.URL, sink RefSink) {
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type == html.TextNode {
|
||||
c.Data = string(RewriteCSS([]byte(c.Data), base, sink))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func attrVal(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if strings.EqualFold(a.Key, key) {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
// Package browser drives a real headless Chrome through the DevTools Protocol so
|
||||
// JavaScript-built pages are captured as they actually render. kage always goes
|
||||
// through here: navigate, let the page settle, then serialise the final DOM —
|
||||
// the same markup a human would have seen — which the rest of the pipeline then
|
||||
// strips of scripts and localises.
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/go-rod/stealth"
|
||||
)
|
||||
|
||||
// Options configure a Pool.
|
||||
type Options struct {
|
||||
Headless bool // run Chrome without a window
|
||||
Workers int // max concurrent pages
|
||||
Settle time.Duration // network-idle quiet period after load
|
||||
RenderTimeout time.Duration // hard cap per page render
|
||||
Scroll bool // auto-scroll to trigger lazy-loaded media
|
||||
ChromeBin string // explicit binary; empty = autodetect
|
||||
ControlURL string // attach to an existing Chrome instead of launching
|
||||
}
|
||||
|
||||
// DefaultOptions returns the baseline render settings.
|
||||
func DefaultOptions() Options {
|
||||
return Options{
|
||||
Headless: true,
|
||||
Workers: 4,
|
||||
Settle: 1500 * time.Millisecond,
|
||||
RenderTimeout: 30 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// Pool owns one Chrome process shared across a run and bounds the number of
|
||||
// pages open at once.
|
||||
type Pool struct {
|
||||
opts Options
|
||||
sem chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
browser *rod.Browser
|
||||
closed bool
|
||||
}
|
||||
|
||||
// New creates a Pool. Chrome is launched lazily on the first Render.
|
||||
func New(opts Options) *Pool {
|
||||
if opts.Workers < 1 {
|
||||
opts.Workers = 1
|
||||
}
|
||||
return &Pool{opts: opts, sem: make(chan struct{}, opts.Workers)}
|
||||
}
|
||||
|
||||
// RenderResult is the outcome of rendering one page.
|
||||
type RenderResult struct {
|
||||
HTML string // the serialised final DOM
|
||||
FinalURL string // URL after any client-side redirects
|
||||
Title string
|
||||
}
|
||||
|
||||
// Render navigates to rawURL, lets it settle, and returns the final rendered
|
||||
// HTML. It acquires a page slot from the pool and releases it when done.
|
||||
func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error) {
|
||||
select {
|
||||
case p.sem <- struct{}{}:
|
||||
defer func() { <-p.sem }()
|
||||
case <-ctx.Done():
|
||||
return RenderResult{}, ctx.Err()
|
||||
}
|
||||
|
||||
b, err := p.getBrowser()
|
||||
if err != nil {
|
||||
return RenderResult{}, err
|
||||
}
|
||||
|
||||
page, err := stealth.Page(b)
|
||||
if err != nil {
|
||||
return RenderResult{}, fmt.Errorf("new page: %w", err)
|
||||
}
|
||||
defer func() { _ = page.Close() }()
|
||||
|
||||
page = page.Context(ctx).Timeout(p.opts.RenderTimeout)
|
||||
|
||||
if err := page.Navigate(rawURL); err != nil {
|
||||
return RenderResult{}, fmt.Errorf("navigate %s: %w", rawURL, err)
|
||||
}
|
||||
if err := page.WaitLoad(); err != nil {
|
||||
return RenderResult{}, fmt.Errorf("wait load %s: %w", rawURL, err)
|
||||
}
|
||||
settle(page, p.opts.Settle)
|
||||
if p.opts.Scroll {
|
||||
autoScroll(page)
|
||||
settle(page, p.opts.Settle)
|
||||
}
|
||||
|
||||
html, err := page.HTML()
|
||||
if err != nil {
|
||||
return RenderResult{}, fmt.Errorf("serialise %s: %w", rawURL, err)
|
||||
}
|
||||
|
||||
res := RenderResult{HTML: html, FinalURL: rawURL}
|
||||
if info, err := page.Info(); err == nil && info != nil {
|
||||
res.FinalURL = info.URL
|
||||
res.Title = info.Title
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// getBrowser lazily connects to or launches Chrome.
|
||||
func (p *Pool) getBrowser() (*rod.Browser, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.closed {
|
||||
return nil, fmt.Errorf("pool is closed")
|
||||
}
|
||||
if p.browser != nil {
|
||||
return p.browser, nil
|
||||
}
|
||||
|
||||
controlURL := p.opts.ControlURL
|
||||
if controlURL == "" {
|
||||
l := launcher.New().
|
||||
Headless(p.opts.Headless).
|
||||
Set("disable-blink-features", "AutomationControlled").
|
||||
Set("disable-dev-shm-usage", "").
|
||||
Set("no-sandbox", "").
|
||||
Set("disable-gpu", "")
|
||||
if bin := p.chromeBin(); bin != "" {
|
||||
l = l.Bin(bin)
|
||||
}
|
||||
u, err := l.Launch()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("launch Chrome: %w", err)
|
||||
}
|
||||
controlURL = u
|
||||
}
|
||||
|
||||
b := rod.New().ControlURL(controlURL)
|
||||
if err := b.Connect(); err != nil {
|
||||
return nil, fmt.Errorf("connect Chrome: %w", err)
|
||||
}
|
||||
p.browser = b
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Close shuts down the managed Chrome process.
|
||||
func (p *Pool) Close() error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.closed = true
|
||||
if p.browser == nil {
|
||||
return nil
|
||||
}
|
||||
err := p.browser.Close()
|
||||
p.browser = nil
|
||||
return err
|
||||
}
|
||||
|
||||
// LookChrome reports the path of a usable Chrome/Chromium binary and whether one
|
||||
// was found, checking KAGE_CHROME, CHROME_BIN, rod's own lookup, and the common
|
||||
// system install locations. Tests use it to skip when no browser is present.
|
||||
func LookChrome() (string, bool) {
|
||||
for _, env := range []string{"KAGE_CHROME", "CHROME_BIN"} {
|
||||
if v := os.Getenv(env); v != "" {
|
||||
return v, true
|
||||
}
|
||||
}
|
||||
if bin, ok := launcher.LookPath(); ok {
|
||||
return bin, true
|
||||
}
|
||||
for _, c := range systemChromeCandidates() {
|
||||
if _, err := os.Stat(c); err == nil {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// chromeBin returns an explicit Chrome path from options or the environment, or
|
||||
// "" to let the launcher find/download one.
|
||||
func (p *Pool) chromeBin() string {
|
||||
if p.opts.ChromeBin != "" {
|
||||
return p.opts.ChromeBin
|
||||
}
|
||||
for _, env := range []string{"KAGE_CHROME", "CHROME_BIN"} {
|
||||
if v := os.Getenv(env); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
for _, c := range systemChromeCandidates() {
|
||||
if _, err := os.Stat(c); err == nil {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func systemChromeCandidates() []string {
|
||||
switch runtime.GOOS {
|
||||
case "darwin":
|
||||
return []string{
|
||||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
||||
}
|
||||
case "windows":
|
||||
return []string{
|
||||
`C:\Program Files\Google\Chrome\Application\chrome.exe`,
|
||||
`C:\Program Files (x86)\Google\Chrome\Application\chrome.exe`,
|
||||
}
|
||||
default:
|
||||
return []string{
|
||||
"/usr/bin/google-chrome",
|
||||
"/usr/bin/google-chrome-stable",
|
||||
"/usr/bin/chromium",
|
||||
"/usr/bin/chromium-browser",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// settle waits for the network to go quiet for d, recovering from any rod
|
||||
// panic and capping the wait so a chatty page can never hang the worker.
|
||||
func settle(page *rod.Page, d time.Duration) {
|
||||
if d <= 0 {
|
||||
return
|
||||
}
|
||||
defer func() { _ = recover() }()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer func() { _ = recover(); close(done) }()
|
||||
wait := page.WaitRequestIdle(d, nil, nil, []proto.NetworkResourceType{})
|
||||
wait()
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(d + 5*time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
// autoScroll scrolls to the bottom in steps to trigger lazy-loaded images.
|
||||
func autoScroll(page *rod.Page) {
|
||||
defer func() { _ = recover() }()
|
||||
_, _ = page.Eval(`() => new Promise((resolve) => {
|
||||
let total = 0;
|
||||
const step = 800;
|
||||
const timer = setInterval(() => {
|
||||
window.scrollBy(0, step);
|
||||
total += step;
|
||||
if (total >= document.body.scrollHeight) {
|
||||
clearInterval(timer);
|
||||
window.scrollTo(0, 0);
|
||||
resolve(true);
|
||||
}
|
||||
}, 100);
|
||||
})`)
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLookChromeReadsEnv(t *testing.T) {
|
||||
t.Setenv("KAGE_CHROME", "/custom/chrome")
|
||||
bin, ok := LookChrome()
|
||||
if !ok || bin != "/custom/chrome" {
|
||||
t.Fatalf("LookChrome() = %q, %v; want /custom/chrome, true", bin, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCapturesFinalDOM(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("render test drives Chrome; skipped under -short")
|
||||
}
|
||||
if _, ok := LookChrome(); !ok {
|
||||
t.Skip("no Chrome/Chromium found; skipping render test")
|
||||
}
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
// A page whose visible content is built by JavaScript: only a real
|
||||
// browser render captures the injected node.
|
||||
_, _ = w.Write([]byte(`<!doctype html><html><body>
|
||||
<div id="app"></div>
|
||||
<script>document.getElementById("app").textContent = "rendered-by-js";</script>
|
||||
</body></html>`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(Options{Headless: true, Workers: 1, Settle: 300 * time.Millisecond, RenderTimeout: 20 * time.Second})
|
||||
defer func() { _ = p.Close() }()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
res, err := p.Render(ctx, srv.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("render: %v", err)
|
||||
}
|
||||
if !strings.Contains(res.HTML, "rendered-by-js") {
|
||||
t.Errorf("render did not capture the JS-built DOM:\n%s", res.HTML)
|
||||
}
|
||||
}
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/tamnd/kage/clone"
|
||||
"github.com/tamnd/kage/urlx"
|
||||
)
|
||||
|
||||
// cloneFlags holds the parsed flag values for one invocation.
|
||||
type cloneFlags struct {
|
||||
out string
|
||||
reserved string
|
||||
workers int
|
||||
assetWorkers int
|
||||
browserPages int
|
||||
maxPages int
|
||||
maxDepth int
|
||||
traversal string
|
||||
maxAssetMB int64
|
||||
timeout time.Duration
|
||||
settle time.Duration
|
||||
renderTO time.Duration
|
||||
scroll bool
|
||||
userAgent string
|
||||
subdomains bool
|
||||
scopePrefix string
|
||||
exclude []string
|
||||
noRobots bool
|
||||
noSitemap bool
|
||||
headful bool
|
||||
keepNoscript bool
|
||||
chromeBin string
|
||||
controlURL string
|
||||
noResume bool
|
||||
force bool
|
||||
quiet bool
|
||||
}
|
||||
|
||||
func newCloneCmd() *cobra.Command {
|
||||
f := &cloneFlags{}
|
||||
cmd := &cobra.Command{
|
||||
Use: "clone <url>",
|
||||
Short: "Clone a site into a self-contained offline folder",
|
||||
Long: "clone fetches the seed URL, follows in-scope links, and writes a browsable\n" +
|
||||
"mirror to <out>/<host>/. Every page is rendered in Chrome and stripped of\n" +
|
||||
"scripts; CSS, images, and fonts are downloaded and rewritten to local paths.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runClone(cmd.Context(), args[0], f)
|
||||
},
|
||||
}
|
||||
fs := cmd.Flags()
|
||||
fs.StringVarP(&f.out, "out", "o", "kage-out", "output root; the mirror lands in <out>/<host>/")
|
||||
fs.StringVar(&f.reserved, "reserved", urlx.DefaultReserved, "reserved dir for assets and state")
|
||||
fs.IntVar(&f.workers, "workers", 4, "concurrent page render workers")
|
||||
fs.IntVar(&f.assetWorkers, "asset-workers", 8, "concurrent asset download workers")
|
||||
fs.IntVar(&f.browserPages, "browser-pages", 4, "Chrome page-pool size")
|
||||
fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "stop after N pages (0 = unlimited)")
|
||||
fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)")
|
||||
fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order: bfs or dfs")
|
||||
fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB")
|
||||
fs.DurationVar(&f.timeout, "timeout", 30*time.Second, "per-request timeout")
|
||||
fs.DurationVar(&f.settle, "settle", 1500*time.Millisecond, "network-idle quiet period before snapshot")
|
||||
fs.DurationVar(&f.renderTO, "render-timeout", 30*time.Second, "hard cap per page render")
|
||||
fs.BoolVar(&f.scroll, "scroll", false, "auto-scroll each page to trigger lazy loading")
|
||||
fs.StringVar(&f.userAgent, "user-agent", clone.DefaultUserAgent, "User-Agent for asset and robots fetches")
|
||||
fs.BoolVar(&f.subdomains, "subdomains", false, "treat subdomains of the seed host as in scope")
|
||||
fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix")
|
||||
fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)")
|
||||
fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)")
|
||||
fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml")
|
||||
fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)")
|
||||
fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap <noscript> content instead of dropping it")
|
||||
fs.StringVar(&f.chromeBin, "chrome", "", "path to the Chrome/Chromium binary")
|
||||
fs.StringVar(&f.controlURL, "control-url", "", "attach to an existing Chrome DevTools endpoint")
|
||||
fs.BoolVar(&f.noResume, "no-resume", false, "do not reuse or write resume state")
|
||||
fs.BoolVarP(&f.force, "force", "f", false, "delete any existing mirror for the host first")
|
||||
fs.BoolVarP(&f.quiet, "quiet", "q", false, "suppress per-page progress lines")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runClone(ctx context.Context, arg string, f *cloneFlags) error {
|
||||
seed, err := urlx.ParseSeed(arg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid url %q: %w", arg, err)
|
||||
}
|
||||
|
||||
cfg := clone.DefaultConfig()
|
||||
cfg.OutDir = f.out
|
||||
cfg.Reserved = f.reserved
|
||||
cfg.Workers = f.workers
|
||||
cfg.AssetWorkers = f.assetWorkers
|
||||
cfg.BrowserPages = f.browserPages
|
||||
cfg.MaxPages = f.maxPages
|
||||
cfg.MaxDepth = f.maxDepth
|
||||
cfg.Traversal = f.traversal
|
||||
cfg.MaxAssetBytes = f.maxAssetMB << 20
|
||||
cfg.Timeout = f.timeout
|
||||
cfg.Settle = f.settle
|
||||
cfg.RenderTimeout = f.renderTO
|
||||
cfg.Scroll = f.scroll
|
||||
cfg.UserAgent = f.userAgent
|
||||
cfg.IncludeSubdomains = f.subdomains
|
||||
cfg.ScopePrefix = f.scopePrefix
|
||||
cfg.ExcludePaths = f.exclude
|
||||
cfg.RespectRobots = !f.noRobots
|
||||
cfg.FollowSitemap = !f.noSitemap
|
||||
cfg.Headless = !f.headful
|
||||
cfg.KeepNoscript = f.keepNoscript
|
||||
cfg.ChromeBin = f.chromeBin
|
||||
cfg.ControlURL = f.controlURL
|
||||
cfg.Resume = !f.noResume
|
||||
cfg.Force = f.force
|
||||
|
||||
logf := func(format string, args ...any) {
|
||||
if !f.quiet {
|
||||
fmt.Fprintln(os.Stderr, styleDim.Render(fmt.Sprintf(format, args...)))
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintln(os.Stderr, styleTitle.Render("kage")+" cloning "+styleAccent.Render(seed.String()))
|
||||
|
||||
c := clone.New(seed, cfg, logf)
|
||||
|
||||
// Live progress ticker on a second line, refreshed every second.
|
||||
tickCtx, stopTicker := context.WithCancel(ctx)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
if f.quiet {
|
||||
return
|
||||
}
|
||||
t := time.NewTicker(time.Second)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-tickCtx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
p := c.Snapshot()
|
||||
fmt.Fprintf(os.Stderr, "\r%s", progressLine(p))
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
res, runErr := c.Run(ctx)
|
||||
stopTicker()
|
||||
<-done
|
||||
if !f.quiet {
|
||||
fmt.Fprint(os.Stderr, "\r\033[K")
|
||||
}
|
||||
|
||||
printSummary(res)
|
||||
|
||||
if runErr != nil && !errors.Is(runErr, context.Canceled) {
|
||||
return runErr
|
||||
}
|
||||
if errors.Is(runErr, context.Canceled) {
|
||||
fmt.Fprintln(os.Stderr, styleWarn.Render("interrupted; resume state saved (rerun to continue)"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// progressLine renders the single-line live counter.
|
||||
func progressLine(p clone.Progress) string {
|
||||
return styleDim.Render(fmt.Sprintf("pages %d assets %d errors %d skipped %d",
|
||||
p.Pages, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
|
||||
}
|
||||
|
||||
// printSummary prints the final tally and where the mirror landed.
|
||||
func printSummary(res clone.Result) {
|
||||
fmt.Fprintln(os.Stderr, styleOK.Render("done")+" "+styleTitle.Render(res.OutDir))
|
||||
fmt.Fprintf(os.Stderr, " %s %d %s %d\n",
|
||||
styleAccent.Render("pages"), res.Pages,
|
||||
styleAccent.Render("assets"), res.Assets)
|
||||
if res.PageErrors+res.AssetErrors > 0 {
|
||||
fmt.Fprintf(os.Stderr, " %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors)
|
||||
}
|
||||
if res.Skipped > 0 {
|
||||
fmt.Fprintf(os.Stderr, " %s %d\n", styleWarn.Render("skipped"), res.Skipped)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " open %s\n", styleAccent.Render("kage serve "+res.OutDir))
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
// Package cli wires kage's command surface: the cobra tree, the global flags,
|
||||
// and the fang-rendered help and errors. The actual work lives in the clone,
|
||||
// browser, sanitize, asset, and urlx packages; this layer only parses flags and
|
||||
// prints progress.
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/charmbracelet/fang"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Execute builds the root command and runs it through fang. main passes the
|
||||
// signal-aware context so Ctrl-C cancels the in-flight clone and flushes resume
|
||||
// state. It returns the process exit code.
|
||||
func Execute(ctx context.Context) int {
|
||||
root := newRoot()
|
||||
opts := []fang.Option{
|
||||
fang.WithVersion(Version),
|
||||
}
|
||||
if err := fang.Execute(ctx, root, opts...); err != nil {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// newRoot assembles the command tree.
|
||||
func newRoot() *cobra.Command {
|
||||
root := &cobra.Command{
|
||||
Use: "kage",
|
||||
Short: "Clone any website for offline viewing, with the JavaScript stripped out",
|
||||
Long: "kage (影, \"shadow\") renders each page in headless Chrome, snapshots the\n" +
|
||||
"final DOM, removes every script and event handler, and localises the CSS,\n" +
|
||||
"images, and fonts so the saved copy looks like the live site but runs no\n" +
|
||||
"code. The result is a plain folder you can open straight from disk.",
|
||||
Version: fmt.Sprintf("%s (commit %s, built %s)", Version, Commit, Date),
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true,
|
||||
}
|
||||
root.AddCommand(newCloneCmd())
|
||||
root.AddCommand(newServeCmd())
|
||||
return root
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func newServeCmd() *cobra.Command {
|
||||
var addr string
|
||||
cmd := &cobra.Command{
|
||||
Use: "serve [dir]",
|
||||
Short: "Preview a cloned folder in your browser",
|
||||
Long: "serve runs a local static file server over a cloned folder so you can click\n" +
|
||||
"through the mirror exactly as a visitor would. With no dir it serves the\n" +
|
||||
"current directory.",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
dir := "."
|
||||
if len(args) == 1 {
|
||||
dir = args[0]
|
||||
}
|
||||
return runServe(dir, addr)
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVarP(&addr, "addr", "a", "127.0.0.1:8800", "address to listen on")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runServe(dir, addr string) error {
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot serve %q: %w", dir, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("%q is not a directory", dir)
|
||||
}
|
||||
abs, _ := filepath.Abs(dir)
|
||||
|
||||
ln, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot listen on %s: %w", addr, err)
|
||||
}
|
||||
|
||||
srv := &http.Server{Handler: http.FileServer(http.Dir(abs))}
|
||||
fmt.Fprintln(os.Stderr, styleTitle.Render("kage serve")+" "+styleDim.Render(abs))
|
||||
fmt.Fprintln(os.Stderr, " open "+styleAccent.Render("http://"+ln.Addr().String()))
|
||||
fmt.Fprintln(os.Stderr, styleDim.Render(" press Ctrl-C to stop"))
|
||||
return srv.Serve(ln)
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package cli
|
||||
|
||||
import "charm.land/lipgloss/v2"
|
||||
|
||||
// Styles for the human-readable progress and summary lines. They degrade to
|
||||
// plain text when the terminal has no colour profile.
|
||||
var (
|
||||
styleTitle = lipgloss.NewStyle().Bold(true)
|
||||
styleAccent = lipgloss.NewStyle().Foreground(lipgloss.Color("12"))
|
||||
styleOK = lipgloss.NewStyle().Foreground(lipgloss.Color("10"))
|
||||
styleWarn = lipgloss.NewStyle().Foreground(lipgloss.Color("11"))
|
||||
styleErr = lipgloss.NewStyle().Foreground(lipgloss.Color("9"))
|
||||
styleDim = lipgloss.NewStyle().Foreground(lipgloss.Color("8"))
|
||||
)
|
||||
@@ -0,0 +1,9 @@
|
||||
package cli
|
||||
|
||||
// Build metadata, stamped via -ldflags at release time. goreleaser targets
|
||||
// github.com/tamnd/kage/cli.{Version,Commit,Date}.
|
||||
var (
|
||||
Version = "dev"
|
||||
Commit = "none"
|
||||
Date = "unknown"
|
||||
)
|
||||
+377
@@ -0,0 +1,377 @@
|
||||
package clone
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/tamnd/kage/asset"
|
||||
"github.com/tamnd/kage/browser"
|
||||
"github.com/tamnd/kage/robots"
|
||||
"github.com/tamnd/kage/sanitize"
|
||||
"github.com/tamnd/kage/urlx"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// Logf is an optional sink for human-readable progress lines.
|
||||
type Logf func(format string, args ...any)
|
||||
|
||||
// Cloner runs one clone. Build it with New, then call Run.
|
||||
type Cloner struct {
|
||||
cfg Config
|
||||
seed *url.URL
|
||||
seedHost string
|
||||
outRoot string
|
||||
statePth string
|
||||
|
||||
pool *browser.Pool
|
||||
dl *asset.Downloader
|
||||
httpC *http.Client
|
||||
robots *robots.Matcher
|
||||
front *frontier
|
||||
stats stats
|
||||
logf Logf
|
||||
|
||||
mu sync.Mutex
|
||||
seenAssets map[string]bool
|
||||
enqueued int // pages offered to the queue
|
||||
wg sync.WaitGroup
|
||||
pageJobs chan pageItem
|
||||
assetJobs chan assetItem
|
||||
}
|
||||
|
||||
type pageItem struct {
|
||||
u *url.URL
|
||||
depth int
|
||||
}
|
||||
|
||||
type assetItem struct {
|
||||
u *url.URL
|
||||
referer string
|
||||
}
|
||||
|
||||
// New builds a Cloner for seed under cfg. It does not touch the network until
|
||||
// Run is called.
|
||||
func New(seed *url.URL, cfg Config, logf Logf) *Cloner {
|
||||
if logf == nil {
|
||||
logf = func(string, ...any) {}
|
||||
}
|
||||
host := seed.Hostname()
|
||||
outRoot := cfg.HostDir(host)
|
||||
return &Cloner{
|
||||
cfg: cfg,
|
||||
seed: seed,
|
||||
seedHost: host,
|
||||
outRoot: outRoot,
|
||||
statePth: filepath.Join(outRoot, cfg.Reserved, "state.json"),
|
||||
dl: asset.NewDownloader(cfg.UserAgent, cfg.Timeout, cfg.MaxAssetBytes),
|
||||
httpC: &http.Client{Timeout: cfg.Timeout},
|
||||
robots: robots.AllowAll(),
|
||||
front: newFrontier(),
|
||||
logf: logf,
|
||||
seenAssets: map[string]bool{},
|
||||
pageJobs: make(chan pageItem),
|
||||
assetJobs: make(chan assetItem),
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot returns the current progress, for a CLI ticker.
|
||||
func (c *Cloner) Snapshot() Progress { return c.stats.snapshot() }
|
||||
|
||||
// Run executes the clone until the frontier drains, MaxPages is hit, or ctx is
|
||||
// cancelled (which flushes the resume state). It returns the final Result.
|
||||
func (c *Cloner) Run(ctx context.Context) (Result, error) {
|
||||
if c.cfg.Force {
|
||||
_ = os.RemoveAll(c.outRoot)
|
||||
}
|
||||
if c.cfg.Resume {
|
||||
if err := c.front.load(c.statePth); err != nil {
|
||||
c.logf("resume: could not load state: %v", err)
|
||||
} else if n := c.front.visitedCount(); n > 0 {
|
||||
c.logf("resume: %d pages already done", n)
|
||||
}
|
||||
}
|
||||
|
||||
c.pool = browser.New(browser.Options{
|
||||
Headless: c.cfg.Headless,
|
||||
Workers: c.cfg.BrowserPages,
|
||||
Settle: c.cfg.Settle,
|
||||
RenderTimeout: c.cfg.RenderTimeout,
|
||||
Scroll: c.cfg.Scroll,
|
||||
ChromeBin: c.cfg.ChromeBin,
|
||||
ControlURL: c.cfg.ControlURL,
|
||||
})
|
||||
defer func() { _ = c.pool.Close() }()
|
||||
|
||||
c.loadRobots(ctx)
|
||||
|
||||
// Start workers.
|
||||
var workers sync.WaitGroup
|
||||
for range max1(c.cfg.Workers) {
|
||||
workers.Go(func() {
|
||||
for j := range c.pageJobs {
|
||||
c.processPage(ctx, j)
|
||||
c.wg.Done()
|
||||
}
|
||||
})
|
||||
}
|
||||
for range max1(c.cfg.AssetWorkers) {
|
||||
workers.Go(func() {
|
||||
for j := range c.assetJobs {
|
||||
c.processAsset(ctx, j)
|
||||
c.wg.Done()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Seed.
|
||||
c.enqueuePage(ctx, c.seed, 0)
|
||||
if c.cfg.FollowSitemap {
|
||||
c.seedSitemaps(ctx)
|
||||
}
|
||||
|
||||
// Close the job channels once every outstanding item is processed.
|
||||
go func() {
|
||||
c.wg.Wait()
|
||||
close(c.pageJobs)
|
||||
close(c.assetJobs)
|
||||
}()
|
||||
workers.Wait()
|
||||
|
||||
if err := c.front.save(c.statePth); err != nil {
|
||||
c.logf("could not save resume state: %v", err)
|
||||
}
|
||||
|
||||
res := Result{Progress: c.stats.snapshot(), OutDir: c.outRoot}
|
||||
if ctx.Err() != nil {
|
||||
return res, ctx.Err()
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// loadRobots fetches and parses robots.txt (unless disabled) and seeds the
|
||||
// sitemap list it advertises.
|
||||
func (c *Cloner) loadRobots(ctx context.Context) {
|
||||
if !c.cfg.RespectRobots {
|
||||
return
|
||||
}
|
||||
robotsURL := c.seed.Scheme + "://" + c.seed.Host + "/robots.txt"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, robotsURL, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", c.cfg.UserAgent)
|
||||
resp, err := c.httpC.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.robots = robots.Parse(string(data), "kage")
|
||||
}
|
||||
|
||||
// seedSitemaps adds in-scope sitemap URLs (from robots and the default path) to
|
||||
// the frontier.
|
||||
func (c *Cloner) seedSitemaps(ctx context.Context) {
|
||||
seeds := append([]string{}, c.robots.Sitemaps...)
|
||||
seeds = append(seeds, c.seed.Scheme+"://"+c.seed.Host+"/sitemap.xml")
|
||||
locs := collectSitemaps(ctx, c.httpC, c.cfg.UserAgent, seeds)
|
||||
added := 0
|
||||
for _, loc := range locs {
|
||||
u, err := urlx.Normalize(c.seed, loc)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if urlx.InScope(c.seed, u, c.cfg.scope()) && urlx.LikelyPage(u) {
|
||||
if c.enqueuePage(ctx, u, 1) {
|
||||
added++
|
||||
}
|
||||
}
|
||||
}
|
||||
if added > 0 {
|
||||
c.logf("sitemap: seeded %d URLs", added)
|
||||
}
|
||||
}
|
||||
|
||||
// processPage renders one page, rewrites its references to local paths, strips
|
||||
// every script, and writes the result.
|
||||
func (c *Cloner) processPage(ctx context.Context, j pageItem) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
key := urlx.Key(j.u)
|
||||
if c.cfg.RespectRobots && !c.robots.Allowed(j.u.Path) {
|
||||
c.stats.skipped.Add(1)
|
||||
return
|
||||
}
|
||||
|
||||
res, err := c.pool.Render(ctx, j.u.String())
|
||||
if err != nil {
|
||||
c.stats.pageErrors.Add(1)
|
||||
c.logf("page error %s: %v", j.u, err)
|
||||
return
|
||||
}
|
||||
|
||||
root, err := html.Parse(strings.NewReader(res.HTML))
|
||||
if err != nil {
|
||||
c.stats.pageErrors.Add(1)
|
||||
c.logf("parse error %s: %v", j.u, err)
|
||||
return
|
||||
}
|
||||
|
||||
localFile := urlx.LocalPath(c.seedHost, j.u, urlx.Page, c.cfg.Reserved)
|
||||
fileDir := urlx.Dir(localFile)
|
||||
|
||||
sink := func(u *url.URL, kind urlx.Kind) string {
|
||||
switch kind {
|
||||
case urlx.Page:
|
||||
if urlx.InScope(c.seed, u, c.cfg.scope()) {
|
||||
c.enqueuePage(ctx, u, j.depth+1)
|
||||
local := urlx.LocalPath(c.seedHost, u, urlx.Page, c.cfg.Reserved)
|
||||
return urlx.Rel(fileDir, local)
|
||||
}
|
||||
return u.String() // external page link stays on the live web
|
||||
default: // Asset
|
||||
c.enqueueAsset(ctx, u, j.u.String())
|
||||
local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
|
||||
return urlx.Rel(fileDir, local)
|
||||
}
|
||||
}
|
||||
|
||||
asset.RewriteHTML(root, j.u, sink)
|
||||
sanitize.CleanTree(root, sanitize.Options{
|
||||
KeepNoscript: c.cfg.KeepNoscript,
|
||||
Banner: "cloned by kage from " + j.u.String(),
|
||||
})
|
||||
|
||||
var buf strings.Builder
|
||||
if err := html.Render(&buf, root); err != nil {
|
||||
c.stats.pageErrors.Add(1)
|
||||
return
|
||||
}
|
||||
if err := c.writeFile(localFile, []byte(buf.String())); err != nil {
|
||||
c.stats.pageErrors.Add(1)
|
||||
c.logf("write error %s: %v", localFile, err)
|
||||
return
|
||||
}
|
||||
c.front.markVisited(key)
|
||||
c.stats.pages.Add(1)
|
||||
}
|
||||
|
||||
// processAsset downloads one asset, rewriting CSS references on the way, and
|
||||
// writes it to its deterministic local path.
|
||||
func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
res, err := c.dl.Get(ctx, j.u, j.referer)
|
||||
if err != nil {
|
||||
c.stats.assetErrors.Add(1)
|
||||
c.logf("asset error %s: %v", j.u, err)
|
||||
return
|
||||
}
|
||||
|
||||
localFile := urlx.LocalPath(c.seedHost, j.u, urlx.Asset, c.cfg.Reserved)
|
||||
body := res.Body
|
||||
if res.IsCSS {
|
||||
fileDir := urlx.Dir(localFile)
|
||||
cssSink := func(u *url.URL, _ urlx.Kind) string {
|
||||
c.enqueueAsset(ctx, u, j.u.String())
|
||||
local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
|
||||
return urlx.Rel(fileDir, local)
|
||||
}
|
||||
body = asset.RewriteCSS(body, j.u, cssSink)
|
||||
}
|
||||
if err := c.writeFile(localFile, body); err != nil {
|
||||
c.stats.assetErrors.Add(1)
|
||||
c.logf("write error %s: %v", localFile, err)
|
||||
return
|
||||
}
|
||||
c.stats.assets.Add(1)
|
||||
}
|
||||
|
||||
// enqueuePage offers a page URL to the frontier, honouring the visited set, the
|
||||
// depth cap, and the page budget. It reports whether the page was newly queued.
|
||||
func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool {
|
||||
if c.cfg.MaxDepth > 0 && depth > c.cfg.MaxDepth {
|
||||
return false
|
||||
}
|
||||
key := urlx.Key(u)
|
||||
if c.front.isVisited(key) {
|
||||
return false
|
||||
}
|
||||
if !c.front.offer(key) {
|
||||
return false
|
||||
}
|
||||
c.mu.Lock()
|
||||
if c.cfg.MaxPages > 0 && c.enqueued >= c.cfg.MaxPages {
|
||||
c.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
c.enqueued++
|
||||
c.mu.Unlock()
|
||||
|
||||
c.wg.Add(1)
|
||||
go func() {
|
||||
select {
|
||||
case c.pageJobs <- pageItem{u: u, depth: depth}:
|
||||
case <-ctx.Done():
|
||||
c.wg.Done()
|
||||
}
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
// enqueueAsset offers an asset URL for download, deduping by canonical URL.
|
||||
func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) {
|
||||
key := urlx.Key(u)
|
||||
c.mu.Lock()
|
||||
if c.seenAssets[key] {
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
c.seenAssets[key] = true
|
||||
c.mu.Unlock()
|
||||
|
||||
c.wg.Add(1)
|
||||
go func() {
|
||||
select {
|
||||
case c.assetJobs <- assetItem{u: u, referer: referer}:
|
||||
case <-ctx.Done():
|
||||
c.wg.Done()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// writeFile writes data to a slash path relative to the mirror root, creating
|
||||
// parent directories. The path is cleaned so it can never escape the root.
|
||||
func (c *Cloner) writeFile(relSlash string, data []byte) error {
|
||||
rel := filepath.FromSlash(relSlash)
|
||||
full := filepath.Join(c.outRoot, rel)
|
||||
if !strings.HasPrefix(full, filepath.Clean(c.outRoot)+string(os.PathSeparator)) && full != c.outRoot {
|
||||
return fmt.Errorf("refusing to write outside mirror root: %s", relSlash)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(full, data, 0o644)
|
||||
}
|
||||
|
||||
func max1(n int) int {
|
||||
if n < 1 {
|
||||
return 1
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package clone
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tamnd/kage/browser"
|
||||
"github.com/tamnd/kage/urlx"
|
||||
)
|
||||
|
||||
// testSite is a tiny two-page site with a stylesheet, an image, an inline
|
||||
// script, an onclick handler, and a javascript: link, so a full clone exercises
|
||||
// rendering, asset localisation, and JavaScript stripping at once.
|
||||
func testSite(t *testing.T) *httptest.Server {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte(`<!doctype html><html><head>
|
||||
<link rel="stylesheet" href="/site.css">
|
||||
<script src="/app.js"></script>
|
||||
</head><body>
|
||||
<h1>Home</h1>
|
||||
<img src="/logo.png" alt="logo">
|
||||
<a href="/about">About</a>
|
||||
<a href="javascript:void(0)" onclick="boom()">Danger</a>
|
||||
<script>console.log("inline")</script>
|
||||
</body></html>`))
|
||||
})
|
||||
mux.HandleFunc("/about", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte(`<!doctype html><html><head>
|
||||
<link rel="stylesheet" href="/site.css">
|
||||
</head><body><h1>About</h1><a href="/">Home</a></body></html>`))
|
||||
})
|
||||
mux.HandleFunc("/site.css", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/css")
|
||||
_, _ = w.Write([]byte(`body{background:url("/bg.png")} h1{color:red}`))
|
||||
})
|
||||
mux.HandleFunc("/app.js", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/javascript")
|
||||
_, _ = w.Write([]byte(`document.body.dataset.ran = "1";`))
|
||||
})
|
||||
mux.HandleFunc("/logo.png", servePNG)
|
||||
mux.HandleFunc("/bg.png", servePNG)
|
||||
mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("User-agent: *\nAllow: /\n"))
|
||||
})
|
||||
return httptest.NewServer(mux)
|
||||
}
|
||||
|
||||
// a 1x1 transparent PNG.
|
||||
var pngBytes = []byte{
|
||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
|
||||
0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
|
||||
0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89, 0x00, 0x00, 0x00,
|
||||
0x0a, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0x00, 0x01, 0x00, 0x00,
|
||||
0x05, 0x00, 0x01, 0x0d, 0x0a, 0x2d, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x49,
|
||||
0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
|
||||
}
|
||||
|
||||
func servePNG(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(pngBytes)
|
||||
}
|
||||
|
||||
func TestCloneEndToEnd(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("clone end-to-end drives Chrome; skipped under -short")
|
||||
}
|
||||
if _, ok := browser.LookChrome(); !ok {
|
||||
t.Skip("no Chrome/Chromium found; skipping clone end-to-end")
|
||||
}
|
||||
|
||||
srv := testSite(t)
|
||||
defer srv.Close()
|
||||
|
||||
seed, err := urlx.ParseSeed(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse seed: %v", err)
|
||||
}
|
||||
|
||||
out := t.TempDir()
|
||||
cfg := DefaultConfig()
|
||||
cfg.OutDir = out
|
||||
cfg.Settle = 300 * time.Millisecond
|
||||
cfg.RenderTimeout = 20 * time.Second
|
||||
cfg.Timeout = 10 * time.Second
|
||||
|
||||
c := New(seed, cfg, t.Logf)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
|
||||
res, err := c.Run(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
if res.Pages < 2 {
|
||||
t.Fatalf("expected at least 2 pages, got %d", res.Pages)
|
||||
}
|
||||
|
||||
root := res.OutDir
|
||||
indexPath := filepath.Join(root, "index.html")
|
||||
body := readFile(t, indexPath)
|
||||
|
||||
// JavaScript is gone: no <script>, no onclick, no javascript: URL.
|
||||
if strings.Contains(strings.ToLower(body), "<script") {
|
||||
t.Error("index.html still contains a <script> tag")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(body), "onclick") {
|
||||
t.Error("index.html still contains an onclick handler")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(body), "javascript:") {
|
||||
t.Error("index.html still contains a javascript: URL")
|
||||
}
|
||||
|
||||
// Layout is preserved: the stylesheet link survives and points local.
|
||||
if !strings.Contains(body, "stylesheet") {
|
||||
t.Error("index.html lost its stylesheet link")
|
||||
}
|
||||
if strings.Contains(body, srv.URL+"/site.css") {
|
||||
t.Error("stylesheet still points at the live origin")
|
||||
}
|
||||
|
||||
// The about page and the localised assets exist on disk.
|
||||
if !fileExists(filepath.Join(root, "about", "index.html")) {
|
||||
t.Error("about page was not written")
|
||||
}
|
||||
assetDir := filepath.Join(root, cfg.Reserved)
|
||||
if !anyFileUnder(t, assetDir, "site.css") {
|
||||
t.Error("site.css was not downloaded")
|
||||
}
|
||||
if !anyFileUnder(t, assetDir, "logo.png") {
|
||||
t.Error("logo.png was not downloaded")
|
||||
}
|
||||
|
||||
// The localised CSS had its url() rewritten away from the origin.
|
||||
css := readAnyFile(t, assetDir, "site.css")
|
||||
if strings.Contains(css, srv.URL) {
|
||||
t.Error("site.css still references the live origin in url()")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneResumeSkipsVisited(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("resume test drives Chrome; skipped under -short")
|
||||
}
|
||||
if _, ok := browser.LookChrome(); !ok {
|
||||
t.Skip("no Chrome/Chromium found; skipping resume test")
|
||||
}
|
||||
|
||||
srv := testSite(t)
|
||||
defer srv.Close()
|
||||
seed, _ := urlx.ParseSeed(srv.URL)
|
||||
|
||||
out := t.TempDir()
|
||||
cfg := DefaultConfig()
|
||||
cfg.OutDir = out
|
||||
cfg.Settle = 300 * time.Millisecond
|
||||
|
||||
c1 := New(seed, cfg, t.Logf)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
if _, err := c1.Run(ctx); err != nil {
|
||||
t.Fatalf("first run: %v", err)
|
||||
}
|
||||
|
||||
// Second run with resume on should find the state and re-render nothing new.
|
||||
c2 := New(seed, cfg, t.Logf)
|
||||
res2, err := c2.Run(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("second run: %v", err)
|
||||
}
|
||||
if res2.Pages != 0 {
|
||||
t.Fatalf("resume should skip all visited pages, but rendered %d", res2.Pages)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func fileExists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func anyFileUnder(t *testing.T, dir, name string) bool {
|
||||
t.Helper()
|
||||
found := false
|
||||
_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() && strings.HasSuffix(p, name) {
|
||||
found = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
func readAnyFile(t *testing.T, dir, name string) string {
|
||||
t.Helper()
|
||||
var out string
|
||||
_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() && strings.HasSuffix(p, name) {
|
||||
b, _ := os.ReadFile(p)
|
||||
out = string(b)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
// Package clone is kage's engine: it ties the Chrome pool, the JavaScript
|
||||
// stripper, the asset localiser, and the URL↔path mapper into one resumable,
|
||||
// polite crawl that turns a live site into a browsable offline folder.
|
||||
package clone
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/tamnd/kage/urlx"
|
||||
)
|
||||
|
||||
// Config is the full set of knobs for a clone run. DefaultConfig fills the
|
||||
// baseline; the CLI overlays flags on top.
|
||||
type Config struct {
|
||||
OutDir string // output root; the mirror lands in <OutDir>/<host>/
|
||||
Reserved string // reserved dir name for assets and state (default "_kage")
|
||||
|
||||
Workers int // page render workers
|
||||
AssetWorkers int // HTTP asset download workers
|
||||
BrowserPages int // Chrome page-pool size
|
||||
MaxPages int // stop after N pages (0 = unlimited)
|
||||
MaxDepth int // BFS/DFS depth cap (0 = unlimited)
|
||||
Traversal string
|
||||
MaxAssetBytes int64
|
||||
|
||||
Timeout time.Duration // per HTTP request
|
||||
Settle time.Duration // network-idle quiet period
|
||||
RenderTimeout time.Duration // hard cap per page render
|
||||
Scroll bool
|
||||
|
||||
UserAgent string
|
||||
IncludeSubdomains bool
|
||||
ScopePrefix string
|
||||
ExcludePaths []string
|
||||
|
||||
RespectRobots bool
|
||||
FollowSitemap bool
|
||||
Headless bool
|
||||
KeepNoscript bool
|
||||
ChromeBin string
|
||||
ControlURL string
|
||||
|
||||
Resume bool
|
||||
Force bool
|
||||
}
|
||||
|
||||
// DefaultUserAgent is a current desktop Chrome UA, used by the asset fetcher and
|
||||
// the robots fetch so a site treats kage like the browser it drives.
|
||||
const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||
|
||||
// DefaultConfig returns the baseline configuration.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
OutDir: "kage-out",
|
||||
Reserved: urlx.DefaultReserved,
|
||||
Workers: 4,
|
||||
AssetWorkers: 8,
|
||||
BrowserPages: 4,
|
||||
MaxAssetBytes: 25 << 20,
|
||||
Traversal: "bfs",
|
||||
Timeout: 30 * time.Second,
|
||||
Settle: 1500 * time.Millisecond,
|
||||
RenderTimeout: 30 * time.Second,
|
||||
UserAgent: DefaultUserAgent,
|
||||
RespectRobots: true,
|
||||
FollowSitemap: true,
|
||||
Headless: true,
|
||||
Resume: true,
|
||||
}
|
||||
}
|
||||
|
||||
// HostDir returns the mirror root for a seed host: <OutDir>/<host>.
|
||||
func (c Config) HostDir(host string) string {
|
||||
return filepath.Join(c.OutDir, host)
|
||||
}
|
||||
|
||||
// scope builds the urlx scope config from the run config.
|
||||
func (c Config) scope() urlx.ScopeConfig {
|
||||
return urlx.ScopeConfig{
|
||||
IncludeSubdomains: c.IncludeSubdomains,
|
||||
ScopePrefix: c.ScopePrefix,
|
||||
ExcludePaths: c.ExcludePaths,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package clone
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// frontier is the deduped set of page URLs kage has already seen. It is small,
|
||||
// concurrency-safe, and persists to disk so --resume can skip work already done.
|
||||
// The actual queueing is handled by the cloner's channels; the frontier only
|
||||
// answers "is this URL new?" and remembers the answer.
|
||||
type frontier struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]bool // queued or visited
|
||||
visited map[string]bool // fully written
|
||||
}
|
||||
|
||||
func newFrontier() *frontier {
|
||||
return &frontier{seen: map[string]bool{}, visited: map[string]bool{}}
|
||||
}
|
||||
|
||||
// offer reports whether key is new (and records it as seen). A repeated key
|
||||
// returns false so it is enqueued only once.
|
||||
func (f *frontier) offer(key string) bool {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.seen[key] {
|
||||
return false
|
||||
}
|
||||
f.seen[key] = true
|
||||
return true
|
||||
}
|
||||
|
||||
// markVisited records that a page was written.
|
||||
func (f *frontier) markVisited(key string) {
|
||||
f.mu.Lock()
|
||||
f.visited[key] = true
|
||||
f.mu.Unlock()
|
||||
}
|
||||
|
||||
// isVisited reports whether a page was already written in a previous run.
|
||||
func (f *frontier) isVisited(key string) bool {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.visited[key]
|
||||
}
|
||||
|
||||
func (f *frontier) visitedCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.visited)
|
||||
}
|
||||
|
||||
// state is the JSON shape persisted for resume.
|
||||
type state struct {
|
||||
Visited []string `json:"visited"`
|
||||
}
|
||||
|
||||
// load reads a previously saved visited set; a missing file is not an error.
|
||||
func (f *frontier) load(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
var s state
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
return err
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, v := range s.Visited {
|
||||
f.visited[v] = true
|
||||
f.seen[v] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// save writes the visited set atomically (write temp, rename).
|
||||
func (f *frontier) save(path string) error {
|
||||
f.mu.Lock()
|
||||
visited := make([]string, 0, len(f.visited))
|
||||
for v := range f.visited {
|
||||
visited = append(visited, v)
|
||||
}
|
||||
f.mu.Unlock()
|
||||
sort.Strings(visited)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(state{Visited: visited}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package clone
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFrontierOfferDedups(t *testing.T) {
|
||||
f := newFrontier()
|
||||
if !f.offer("a") {
|
||||
t.Fatal("first offer of a should be new")
|
||||
}
|
||||
if f.offer("a") {
|
||||
t.Fatal("second offer of a should be a duplicate")
|
||||
}
|
||||
if !f.offer("b") {
|
||||
t.Fatal("first offer of b should be new")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrontierVisited(t *testing.T) {
|
||||
f := newFrontier()
|
||||
if f.isVisited("x") {
|
||||
t.Fatal("x should not be visited yet")
|
||||
}
|
||||
f.markVisited("x")
|
||||
if !f.isVisited("x") {
|
||||
t.Fatal("x should be visited after markVisited")
|
||||
}
|
||||
if f.visitedCount() != 1 {
|
||||
t.Fatalf("visitedCount = %d, want 1", f.visitedCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrontierSaveLoadRoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "sub", "state.json")
|
||||
|
||||
f := newFrontier()
|
||||
f.markVisited("https://example.com/")
|
||||
f.markVisited("https://example.com/about")
|
||||
if err := f.save(path); err != nil {
|
||||
t.Fatalf("save: %v", err)
|
||||
}
|
||||
|
||||
g := newFrontier()
|
||||
if err := g.load(path); err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if g.visitedCount() != 2 {
|
||||
t.Fatalf("loaded visitedCount = %d, want 2", g.visitedCount())
|
||||
}
|
||||
if !g.isVisited("https://example.com/about") {
|
||||
t.Fatal("about should be visited after load")
|
||||
}
|
||||
// A loaded visited URL is also seen, so it is not re-offered.
|
||||
if g.offer("https://example.com/about") {
|
||||
t.Fatal("a loaded URL should not be offered again")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrontierLoadMissingIsNotError(t *testing.T) {
|
||||
f := newFrontier()
|
||||
if err := f.load(filepath.Join(t.TempDir(), "nope.json")); err != nil {
|
||||
t.Fatalf("loading a missing file should be a no-op, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package clone
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// sitemapDoc covers both a urlset and a sitemapindex: both carry <loc> elements,
|
||||
// so one shape extracts the URLs from either.
|
||||
type sitemapDoc struct {
|
||||
URLs []sitemapEntry `xml:"url"`
|
||||
Sitemaps []sitemapEntry `xml:"sitemap"`
|
||||
}
|
||||
|
||||
type sitemapEntry struct {
|
||||
Loc string `xml:"loc"`
|
||||
}
|
||||
|
||||
// fetchSitemap downloads and parses one sitemap, returning page locations and,
|
||||
// for a sitemap index, the child sitemap URLs.
|
||||
func fetchSitemap(ctx context.Context, client *http.Client, ua, sitemapURL string) (locs, children []string) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, sitemapURL, nil)
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
if ua != "" {
|
||||
req.Header.Set("User-Agent", ua)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, nil
|
||||
}
|
||||
var doc sitemapDoc
|
||||
dec := xml.NewDecoder(resp.Body)
|
||||
if err := dec.Decode(&doc); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
for _, u := range doc.URLs {
|
||||
if s := strings.TrimSpace(u.Loc); s != "" {
|
||||
locs = append(locs, s)
|
||||
}
|
||||
}
|
||||
for _, sm := range doc.Sitemaps {
|
||||
if s := strings.TrimSpace(sm.Loc); s != "" {
|
||||
children = append(children, s)
|
||||
}
|
||||
}
|
||||
return locs, children
|
||||
}
|
||||
|
||||
// collectSitemaps walks a set of seed sitemap URLs (following index files one
|
||||
// level deep) and returns all discovered page locations, bounded by a deadline.
|
||||
func collectSitemaps(ctx context.Context, client *http.Client, ua string, seeds []string) []string {
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
queue := append([]string{}, seeds...)
|
||||
for len(queue) > 0 && ctx.Err() == nil {
|
||||
sm := queue[0]
|
||||
queue = queue[1:]
|
||||
if seen[sm] {
|
||||
continue
|
||||
}
|
||||
seen[sm] = true
|
||||
locs, children := fetchSitemap(ctx, client, ua, sm)
|
||||
out = append(out, locs...)
|
||||
for _, c := range children {
|
||||
if !seen[c] {
|
||||
queue = append(queue, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package clone
|
||||
|
||||
import "sync/atomic"
|
||||
|
||||
// stats are the live counters of a run, read by the CLI's progress ticker.
|
||||
type stats struct {
|
||||
pages atomic.Int64
|
||||
assets atomic.Int64
|
||||
pageErrors atomic.Int64
|
||||
assetErrors atomic.Int64
|
||||
skipped atomic.Int64 // robots-disallowed or out of budget
|
||||
}
|
||||
|
||||
// Progress is a snapshot of a run for display.
|
||||
type Progress struct {
|
||||
Pages int64
|
||||
Assets int64
|
||||
PageErrors int64
|
||||
AssetErrors int64
|
||||
Skipped int64
|
||||
}
|
||||
|
||||
func (s *stats) snapshot() Progress {
|
||||
return Progress{
|
||||
Pages: s.pages.Load(),
|
||||
Assets: s.assets.Load(),
|
||||
PageErrors: s.pageErrors.Load(),
|
||||
AssetErrors: s.assetErrors.Load(),
|
||||
Skipped: s.skipped.Load(),
|
||||
}
|
||||
}
|
||||
|
||||
// Result is the final outcome returned by Run.
|
||||
type Result struct {
|
||||
Progress
|
||||
OutDir string
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Command kage clones a website into a self-contained offline folder: it renders
|
||||
// every page in headless Chrome, strips all JavaScript, and localises the CSS,
|
||||
// images, and fonts so the saved copy looks like the live site but runs no code.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/signal"
|
||||
|
||||
"github.com/tamnd/kage/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
defer stop()
|
||||
os.Exit(cli.Execute(ctx))
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
---
|
||||
title: "kage"
|
||||
description: "kage (影, shadow) clones any website into a self-contained folder you can browse offline, with all the JavaScript stripped out. Render in headless Chrome, remove every script, localise the CSS, images, and fonts, from one pure-Go binary."
|
||||
heroTitle: "A website, frozen as a shadow"
|
||||
heroLead: "kage renders every page in headless Chrome, snapshots the final DOM, removes every script and event handler, and downloads and rewrites the CSS, images, and fonts. The result looks like the live site but runs no code: a plain folder of .html files you can open straight from disk."
|
||||
heroPrimaryURL: "/getting-started/quick-start/"
|
||||
heroPrimaryText: "Get started"
|
||||
---
|
||||
|
||||
Saving a page with "Save As" gives you a copy that still phones home, still runs
|
||||
analytics, and often renders blank because the markup is built by JavaScript at
|
||||
runtime. kage (影, "shadow") takes the opposite approach: it drives a real
|
||||
browser, captures the page the way a human would have seen it, then makes it
|
||||
inert.
|
||||
|
||||
```bash
|
||||
kage clone example.com
|
||||
kage serve kage-out/example.com
|
||||
```
|
||||
|
||||
## What it does
|
||||
|
||||
- **Renders first, saves second.** Each page goes through real headless Chrome,
|
||||
so a page whose content is assembled by JavaScript is captured fully, not as
|
||||
an empty shell.
|
||||
- **Strips every script.** Once the DOM is captured, kage removes all `<script>`
|
||||
tags, every `on*` event handler, and any `javascript:` URL. The saved page
|
||||
makes no network calls and runs no code.
|
||||
- **Keeps the layout.** Stylesheets, images, fonts, and media are downloaded and
|
||||
rewritten to relative local paths, so the offline copy looks like the original.
|
||||
- **Stays browsable.** In-scope links are rewritten to point at the other saved
|
||||
pages, so you can click around the mirror exactly as you would the live site.
|
||||
|
||||
## Where to go next
|
||||
|
||||
- New here? Start with the [introduction](/getting-started/introduction/), then
|
||||
the [quick start](/getting-started/quick-start/).
|
||||
- Want to install it? See [installation](/getting-started/installation/).
|
||||
- Looking for a specific task? The [guides](/guides/) cover scoping a crawl,
|
||||
serving a mirror, and resuming an interrupted run.
|
||||
- Need every flag? The [CLI reference](/reference/cli/) is the full surface.
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
title: "Getting started"
|
||||
linkTitle: "Getting started"
|
||||
description: "Install kage and clone your first site into a browsable offline folder in under a minute."
|
||||
weight: 10
|
||||
featured: true
|
||||
---
|
||||
|
||||
Three short pages: how kage thinks about cloning a site (render, strip, localise),
|
||||
how to install the binary and point it at a browser, and a guided first run that
|
||||
ends with a real offline mirror you can click through.
|
||||
@@ -0,0 +1,63 @@
|
||||
---
|
||||
title: "Installation"
|
||||
description: "Install kage from Go, Homebrew, a release archive, a Linux package, or the container image, and point it at a browser."
|
||||
weight: 20
|
||||
---
|
||||
|
||||
kage is a single binary. Pick whichever channel suits you.
|
||||
|
||||
## Go
|
||||
|
||||
```bash
|
||||
go install github.com/tamnd/kage/cmd/kage@latest
|
||||
```
|
||||
|
||||
## Homebrew
|
||||
|
||||
```bash
|
||||
brew install tamnd/tap/kage
|
||||
```
|
||||
|
||||
## Release archives and Linux packages
|
||||
|
||||
Every [release](https://github.com/tamnd/kage/releases) attaches `tar.gz`
|
||||
archives (and a `.zip` for Windows) for Linux, macOS, Windows, and FreeBSD, plus
|
||||
`.deb`, `.rpm`, and `.apk` packages and a `checksums.txt` with a cosign
|
||||
signature. Download the one for your platform, extract `kage`, and put it on your
|
||||
`PATH`.
|
||||
|
||||
```bash
|
||||
# Debian/Ubuntu
|
||||
sudo dpkg -i kage_*_linux_amd64.deb
|
||||
|
||||
# Fedora/RHEL
|
||||
sudo rpm -i kage_*_linux_amd64.rpm
|
||||
```
|
||||
|
||||
## Container
|
||||
|
||||
The image bundles Chromium, so it needs nothing else:
|
||||
|
||||
```bash
|
||||
docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com
|
||||
```
|
||||
|
||||
The mirror lands in `./out/example.com/` on your host.
|
||||
|
||||
## You need a browser
|
||||
|
||||
kage drives a real Chrome to render pages. Outside the container image, it needs
|
||||
Chrome or Chromium available on the machine. It looks for a system install
|
||||
automatically (Google Chrome on macOS and Windows, `google-chrome`/`chromium` on
|
||||
Linux). To use a specific binary:
|
||||
|
||||
```bash
|
||||
kage clone example.com --chrome /path/to/chromium
|
||||
# or
|
||||
export KAGE_CHROME=/path/to/chromium
|
||||
```
|
||||
|
||||
If no browser is found, kage's launcher can download a private copy of Chromium
|
||||
on first use.
|
||||
|
||||
Next: [the quick start](/getting-started/quick-start/).
|
||||
@@ -0,0 +1,53 @@
|
||||
---
|
||||
title: "Introduction"
|
||||
description: "Why kage renders before it saves, and what it means to strip the JavaScript out of a clone."
|
||||
weight: 10
|
||||
---
|
||||
|
||||
A normal website is not a document; it is a program. The HTML the server sends
|
||||
is often a near-empty shell, and the page you actually see is assembled in your
|
||||
browser by JavaScript: fetching data, building the DOM, wiring up handlers. That
|
||||
is why "Save As" so often fails. You get the shell, not the page, and whatever
|
||||
you do get still runs trackers and phones home when you open it.
|
||||
|
||||
kage treats a clone as three steps in order.
|
||||
|
||||
## 1. Render
|
||||
|
||||
Every page is loaded in a real headless Chrome through the DevTools protocol.
|
||||
kage navigates to the URL, waits for the network to go quiet, optionally scrolls
|
||||
to trigger lazy-loaded images, and then serialises the **final** DOM, the markup
|
||||
that exists after the page's JavaScript has finished building it. This is the
|
||||
same thing you would see if you opened the page and chose "Inspect".
|
||||
|
||||
## 2. Strip
|
||||
|
||||
From that captured DOM, kage removes everything executable:
|
||||
|
||||
- every `<script>` tag, inline or external;
|
||||
- every `on*` event handler attribute (`onclick`, `onload`, and the rest);
|
||||
- every `javascript:` URL;
|
||||
- `<meta http-equiv="refresh">` redirects and dead resource hints like
|
||||
`<link rel="preload" as="script">`.
|
||||
|
||||
What remains is inert. The saved page makes no network calls, runs no code, and
|
||||
tracks nothing.
|
||||
|
||||
## 3. Localise
|
||||
|
||||
A page with no working CSS or images is not much of a clone, so kage keeps the
|
||||
parts that define how it looks. It downloads every stylesheet, image, font, and
|
||||
media file, rewrites the references in the HTML and inside the CSS (`url()` and
|
||||
`@import`) to relative local paths, and rewrites in-scope page links to point at
|
||||
the other saved pages. The mirror is fully self-contained: you can move the
|
||||
folder anywhere, open it with no network, and click around.
|
||||
|
||||
## The shape of a clone
|
||||
|
||||
kage crawls breadth-first from a seed URL, staying within the seed's host (and
|
||||
optionally its subdomains). It is polite by default: it honours `robots.txt` and
|
||||
seeds itself from `sitemap.xml`. Output lands in `kage-out/<host>/`, with pages
|
||||
as `<path>/index.html` and assets under a reserved `_kage/` directory alongside
|
||||
the crawl state that powers `--resume`.
|
||||
|
||||
Next: [install kage](/getting-started/installation/).
|
||||
@@ -0,0 +1,68 @@
|
||||
---
|
||||
title: "Quick start"
|
||||
description: "From an empty terminal to a self-contained offline mirror you can click through."
|
||||
weight: 30
|
||||
---
|
||||
|
||||
This walks the core loop: clone a small site, look at what landed on disk, and
|
||||
serve it back so links and assets resolve the way they would on a real host.
|
||||
|
||||
## 1. Clone a site
|
||||
|
||||
```bash
|
||||
kage clone example.com
|
||||
```
|
||||
|
||||
kage launches headless Chrome, renders the home page, strips its scripts, and
|
||||
follows in-scope links breadth-first. A live counter shows pages, assets, and
|
||||
errors as it goes; the final summary tells you where the mirror landed.
|
||||
|
||||
```
|
||||
kage cloning https://example.com
|
||||
done kage-out/example.com
|
||||
pages 12 assets 38
|
||||
open kage serve kage-out/example.com
|
||||
```
|
||||
|
||||
## 2. Look at what landed
|
||||
|
||||
```bash
|
||||
ls kage-out/example.com
|
||||
```
|
||||
|
||||
```
|
||||
index.html # the home page, scripts stripped
|
||||
about/index.html # /about
|
||||
_kage/ # localised assets and crawl state
|
||||
```
|
||||
|
||||
Open `index.html` directly in a browser and it renders offline, with no network.
|
||||
Grep it and you will find no `<script>`, no `onclick`, no `javascript:`.
|
||||
|
||||
## 3. Serve it back
|
||||
|
||||
Opening files directly works, but some sites use root-relative links. `kage
|
||||
serve` runs a local static server so everything resolves exactly as it would
|
||||
live:
|
||||
|
||||
```bash
|
||||
kage serve kage-out/example.com
|
||||
# open http://127.0.0.1:8800
|
||||
```
|
||||
|
||||
## 4. Scope a bigger crawl
|
||||
|
||||
For a large site, bound the crawl so it does not run away:
|
||||
|
||||
```bash
|
||||
# Just the docs section, three levels deep, at most 200 pages
|
||||
kage clone example.com --scope-prefix /docs --max-depth 3 --max-pages 200
|
||||
```
|
||||
|
||||
If you stop a run with Ctrl-C, kage saves its state. Run the same command again
|
||||
and it resumes, skipping the pages it already wrote.
|
||||
|
||||
## Where to go next
|
||||
|
||||
- The [guides](/guides/) cover scoping, serving, and resuming in depth.
|
||||
- The [CLI reference](/reference/cli/) lists every flag.
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
title: "Guides"
|
||||
linkTitle: "Guides"
|
||||
description: "Task-oriented walkthroughs for the things people actually do with kage: scoping a crawl, serving a mirror, and resuming a run."
|
||||
weight: 20
|
||||
featured: true
|
||||
---
|
||||
|
||||
Each guide is built around a job rather than a flag: keeping a crawl inside the
|
||||
lines, viewing what you cloned, and picking up an interrupted run. They assume
|
||||
you have worked through the [quick start](/getting-started/quick-start/).
|
||||
@@ -0,0 +1,48 @@
|
||||
---
|
||||
title: "Resuming a run"
|
||||
description: "Pick up an interrupted clone where it left off, and start fresh when you want to."
|
||||
weight: 30
|
||||
---
|
||||
|
||||
Cloning a large site can take a while, and runs get interrupted: you press
|
||||
Ctrl-C, your laptop sleeps, the network drops. kage is built to pick up where it
|
||||
left off.
|
||||
|
||||
## How resume works
|
||||
|
||||
As it writes each page, kage records it in a small state file inside the mirror,
|
||||
at `<host>/_kage/state.json`. When a run ends, for any reason, that file holds
|
||||
the set of pages already written. Resume is **on by default**: the next time you
|
||||
run the same clone, kage loads the state and skips every page it already wrote,
|
||||
re-crawling only what is left.
|
||||
|
||||
```bash
|
||||
kage clone example.com
|
||||
# ... press Ctrl-C partway through ...
|
||||
# interrupted; resume state saved (rerun to continue)
|
||||
|
||||
kage clone example.com
|
||||
# resume: 137 pages already done
|
||||
```
|
||||
|
||||
Ctrl-C is a clean stop: kage cancels in-flight renders, flushes the state file,
|
||||
and exits. You will not lose the pages already written.
|
||||
|
||||
## Start fresh
|
||||
|
||||
To ignore any previous run and rebuild the mirror from scratch, delete the
|
||||
existing host folder first with `--force`:
|
||||
|
||||
```bash
|
||||
kage clone example.com --force
|
||||
```
|
||||
|
||||
This removes `kage-out/example.com/` before crawling, so nothing from a prior run
|
||||
carries over.
|
||||
|
||||
To run without reading or writing any resume state at all, for a strictly
|
||||
one-shot clone, use `--no-resume`:
|
||||
|
||||
```bash
|
||||
kage clone example.com --no-resume
|
||||
```
|
||||
@@ -0,0 +1,73 @@
|
||||
---
|
||||
title: "Scoping a crawl"
|
||||
description: "Keep a clone inside the lines with depth, page, prefix, subdomain, and exclude controls."
|
||||
weight: 10
|
||||
---
|
||||
|
||||
By default kage crawls every in-scope page it can reach from the seed, staying on
|
||||
the seed's host. On a large site that can be a lot of pages. These flags bound
|
||||
the crawl.
|
||||
|
||||
## Limit by count and depth
|
||||
|
||||
```bash
|
||||
# Stop after 200 pages
|
||||
kage clone example.com --max-pages 200
|
||||
|
||||
# Only follow links three hops from the seed
|
||||
kage clone example.com --max-depth 3
|
||||
```
|
||||
|
||||
`--max-depth 0` (the default) means unlimited depth; `--max-pages 0` means
|
||||
unlimited pages. Combine them to put a hard ceiling on a run.
|
||||
|
||||
## Limit by path
|
||||
|
||||
To clone just one section of a site, restrict the crawl to a path prefix:
|
||||
|
||||
```bash
|
||||
kage clone example.com --scope-prefix /docs
|
||||
```
|
||||
|
||||
Only pages whose path starts with `/docs` are followed. Assets are still fetched
|
||||
from wherever the page references them, so the section renders correctly.
|
||||
|
||||
To skip parts of a site, exclude path prefixes (repeatable):
|
||||
|
||||
```bash
|
||||
kage clone example.com --exclude /archive --exclude /tags
|
||||
```
|
||||
|
||||
## Subdomains
|
||||
|
||||
By default a clone stays on the exact seed host. To treat subdomains of the seed
|
||||
as in scope, add `--subdomains`:
|
||||
|
||||
```bash
|
||||
kage clone example.com --subdomains
|
||||
```
|
||||
|
||||
Now `blog.example.com` and `docs.example.com` are crawled too, each landing
|
||||
under its own host directory inside the mirror.
|
||||
|
||||
## Politeness
|
||||
|
||||
kage honours `robots.txt` by default and seeds itself from `sitemap.xml`. If you
|
||||
are cloning a site you control, or you have a reason to ignore the robots rules,
|
||||
you can turn them off, but do so responsibly:
|
||||
|
||||
```bash
|
||||
kage clone example.com --no-robots --no-sitemap
|
||||
```
|
||||
|
||||
## Lazy-loaded media
|
||||
|
||||
Sites that load images as you scroll will only have their above-the-fold images
|
||||
captured unless you tell kage to scroll each page:
|
||||
|
||||
```bash
|
||||
kage clone example.com --scroll
|
||||
```
|
||||
|
||||
This makes each render a little slower but captures media that only loads on
|
||||
view.
|
||||
@@ -0,0 +1,48 @@
|
||||
---
|
||||
title: "Serving a mirror"
|
||||
description: "View a cloned folder the way it would render on a real host, with kage serve."
|
||||
weight: 20
|
||||
---
|
||||
|
||||
A clone is a plain folder of files, so the simplest way to view it is to open an
|
||||
`.html` file in your browser. That works for many sites. But some pages use
|
||||
root-relative URLs (`/style.css`, `/img/logo.png`), which only resolve correctly
|
||||
when served from the root of a host. `kage serve` gives you that root.
|
||||
|
||||
## Serve a clone
|
||||
|
||||
```bash
|
||||
kage serve kage-out/example.com
|
||||
```
|
||||
|
||||
```
|
||||
kage serve /…/kage-out/example.com
|
||||
open http://127.0.0.1:8800
|
||||
press Ctrl-C to stop
|
||||
```
|
||||
|
||||
Open the printed URL and click around the mirror exactly as you would the live
|
||||
site. Every in-scope link kage rewrote points at another saved page; every asset
|
||||
resolves to its localised copy.
|
||||
|
||||
## Choose an address
|
||||
|
||||
By default kage serves on `127.0.0.1:8800`. Change it with `--addr`:
|
||||
|
||||
```bash
|
||||
# A different port
|
||||
kage serve kage-out/example.com --addr 127.0.0.1:9000
|
||||
|
||||
# Reachable from other machines on your network (be deliberate about this)
|
||||
kage serve kage-out/example.com --addr 0.0.0.0:8800
|
||||
```
|
||||
|
||||
## Serve the current directory
|
||||
|
||||
With no argument, `kage serve` serves the current directory, which is handy from
|
||||
inside an output folder:
|
||||
|
||||
```bash
|
||||
cd kage-out/example.com
|
||||
kage serve
|
||||
```
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
title: "Reference"
|
||||
linkTitle: "Reference"
|
||||
description: "The complete kage surface: every command, every flag, and every environment variable."
|
||||
weight: 30
|
||||
featured: true
|
||||
---
|
||||
|
||||
The exhaustive reference. The [CLI](/reference/cli/) page lists every command and
|
||||
flag; the [configuration](/reference/configuration/) page covers environment
|
||||
variables and the output layout on disk.
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
title: "CLI reference"
|
||||
description: "Every kage command and flag."
|
||||
weight: 10
|
||||
---
|
||||
|
||||
```
|
||||
kage [command] [flags]
|
||||
```
|
||||
|
||||
Two commands: `clone` fetches a site into an offline folder, `serve` previews
|
||||
one. Run `kage <command> --help` for the canonical, up-to-date list.
|
||||
|
||||
## kage clone
|
||||
|
||||
```
|
||||
kage clone <url> [flags]
|
||||
```
|
||||
|
||||
Renders each page in headless Chrome, strips all JavaScript, localises CSS,
|
||||
images, and fonts, and writes a browsable mirror to `<out>/<host>/`.
|
||||
|
||||
### Output
|
||||
|
||||
| Flag | Default | Meaning |
|
||||
|------|---------|---------|
|
||||
| `-o, --out` | `kage-out` | Output root; the mirror lands in `<out>/<host>/` |
|
||||
| `--reserved` | `_kage` | Reserved directory name for assets and crawl state |
|
||||
| `-f, --force` | `false` | Delete any existing mirror for the host before crawling |
|
||||
| `--no-resume` | `false` | Do not read or write resume state |
|
||||
|
||||
### Scope
|
||||
|
||||
| Flag | Default | Meaning |
|
||||
|------|---------|---------|
|
||||
| `-p, --max-pages` | `0` | Stop after N pages (0 = unlimited) |
|
||||
| `-d, --max-depth` | `0` | Link-follow depth cap (0 = unlimited) |
|
||||
| `--scope-prefix` | | Only crawl pages whose path starts with this prefix |
|
||||
| `--subdomains` | `false` | Treat subdomains of the seed host as in scope |
|
||||
| `--exclude` | | Path prefixes to skip (repeatable) |
|
||||
| `--traversal` | `bfs` | Frontier order: `bfs` or `dfs` |
|
||||
|
||||
### Politeness
|
||||
|
||||
| Flag | Default | Meaning |
|
||||
|------|---------|---------|
|
||||
| `--no-robots` | `false` | Ignore `robots.txt` |
|
||||
| `--no-sitemap` | `false` | Do not seed URLs from `sitemap.xml` |
|
||||
| `--user-agent` | Chrome UA | User-Agent for asset and robots fetches |
|
||||
|
||||
### Rendering
|
||||
|
||||
| Flag | Default | Meaning |
|
||||
|------|---------|---------|
|
||||
| `--scroll` | `false` | Auto-scroll each page to trigger lazy loading |
|
||||
| `--settle` | `1.5s` | Network-idle quiet period before snapshotting the DOM |
|
||||
| `--render-timeout` | `30s` | Hard cap per page render |
|
||||
| `--headful` | `false` | Run Chrome with a visible window (debugging) |
|
||||
| `--chrome` | | Path to the Chrome/Chromium binary |
|
||||
| `--control-url` | | Attach to an existing Chrome DevTools endpoint |
|
||||
| `--keep-noscript` | `false` | Unwrap `<noscript>` content instead of dropping it |
|
||||
|
||||
### Concurrency and limits
|
||||
|
||||
| Flag | Default | Meaning |
|
||||
|------|---------|---------|
|
||||
| `--workers` | `4` | Concurrent page render workers |
|
||||
| `--asset-workers` | `8` | Concurrent asset download workers |
|
||||
| `--browser-pages` | `4` | Chrome page-pool size |
|
||||
| `--max-asset-mb` | `25` | Skip assets larger than N MB |
|
||||
| `--timeout` | `30s` | Per-request timeout |
|
||||
| `-q, --quiet` | `false` | Suppress per-page progress lines |
|
||||
|
||||
## kage serve
|
||||
|
||||
```
|
||||
kage serve [dir] [flags]
|
||||
```
|
||||
|
||||
Runs a local static file server over a cloned folder. With no `dir`, serves the
|
||||
current directory.
|
||||
|
||||
| Flag | Default | Meaning |
|
||||
|------|---------|---------|
|
||||
| `-a, --addr` | `127.0.0.1:8800` | Address to listen on |
|
||||
@@ -0,0 +1,54 @@
|
||||
---
|
||||
title: "Configuration"
|
||||
description: "Environment variables kage reads, and the layout of a cloned mirror on disk."
|
||||
weight: 20
|
||||
---
|
||||
|
||||
kage is configured almost entirely through command-line flags (see the
|
||||
[CLI reference](/reference/cli/)). It reads a couple of environment variables for
|
||||
locating the browser.
|
||||
|
||||
## Environment variables
|
||||
|
||||
| Variable | Meaning |
|
||||
|----------|---------|
|
||||
| `KAGE_CHROME` | Path to the Chrome/Chromium binary. Takes precedence over autodetection. Equivalent to `--chrome`. |
|
||||
| `CHROME_BIN` | Fallback Chrome path, read if `KAGE_CHROME` is unset. |
|
||||
|
||||
If neither is set and no system Chrome is found in the usual install locations,
|
||||
kage's launcher can download a private copy of Chromium on first use.
|
||||
|
||||
## Output layout
|
||||
|
||||
A clone of `example.com` lands under `kage-out/example.com/`:
|
||||
|
||||
```
|
||||
kage-out/example.com/
|
||||
├── index.html # the home page (/), scripts stripped
|
||||
├── about/index.html # /about
|
||||
├── blog/
|
||||
│ ├── index.html # /blog
|
||||
│ └── a-post/index.html # /blog/a-post
|
||||
├── _kage/ # reserved directory
|
||||
│ ├── example.com/
|
||||
│ │ ├── site.css # localised stylesheet, url() rewritten
|
||||
│ │ ├── logo.png
|
||||
│ │ └── fonts/body.woff2
|
||||
│ ├── cdn.example.com/ # assets from other hosts, by host
|
||||
│ └── state.json # visited set, for --resume
|
||||
└── ...
|
||||
```
|
||||
|
||||
Key points:
|
||||
|
||||
- **Pages become directories.** A page at `/about` is written as
|
||||
`about/index.html`, so a link to `/about` resolves to a real file when served.
|
||||
- **Assets live under the reserved directory.** Everything kage downloads, CSS,
|
||||
images, fonts, media, goes under `_kage/<asset-host>/`, mirroring the path it
|
||||
had on its origin. Cross-origin assets are grouped by their own host.
|
||||
- **Query strings are folded into the filename.** An asset like
|
||||
`style.css?v=3` is saved with a short hash suffix so two versions never
|
||||
collide.
|
||||
- **State lives in the mirror.** `_kage/state.json` records every page written,
|
||||
which is what makes `--resume` able to skip completed work. Rename the reserved
|
||||
directory with `--reserved` if `_kage` would clash with a real path on the site.
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><rect width="16" height="16" rx="3" fill="#0b6e4f"/><path fill="#f6f8fa" d="M7 2.5a4.5 4.5 0 1 0 2.62 8.15l2.36 2.37a.75.75 0 1 0 1.06-1.06l-2.37-2.36A4.5 4.5 0 0 0 7 2.5Zm-3 4.5a3 3 0 1 1 6 0 3 3 0 0 1-6 0Z"/></svg>
|
||||
|
After Width: | Height: | Size: 300 B |
@@ -0,0 +1,10 @@
|
||||
title = "kage"
|
||||
baseURL = "https://kage.tamnd.com/"
|
||||
description = "kage (影, shadow) clones any website into a self-contained folder you can browse offline, with all the JavaScript stripped out. It renders every page in headless Chrome, removes every script, and localises the CSS, images, and fonts."
|
||||
contentDir = "content"
|
||||
outputDir = "public"
|
||||
theme = "tago-doks"
|
||||
syntaxHighlight = "true"
|
||||
|
||||
[params]
|
||||
github = "https://github.com/tamnd/kage"
|
||||
@@ -0,0 +1,44 @@
|
||||
module github.com/tamnd/kage
|
||||
|
||||
go 1.26.3
|
||||
|
||||
require (
|
||||
charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410
|
||||
github.com/charmbracelet/fang v1.0.0
|
||||
github.com/go-rod/rod v0.116.2
|
||||
github.com/go-rod/stealth v0.4.9
|
||||
github.com/spf13/cobra v1.10.2
|
||||
golang.org/x/net v0.56.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/charmbracelet/colorprofile v0.3.3 // indirect
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20251106190538-99ea45596692 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.11.0 // indirect
|
||||
github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444 // indirect
|
||||
github.com/charmbracelet/x/term v0.2.2 // indirect
|
||||
github.com/charmbracelet/x/termios v0.1.1 // indirect
|
||||
github.com/charmbracelet/x/windows v0.2.2 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.4.1 // indirect
|
||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/mango v0.1.0 // indirect
|
||||
github.com/muesli/mango-cobra v1.2.0 // indirect
|
||||
github.com/muesli/mango-pflag v0.1.0 // indirect
|
||||
github.com/muesli/roff v0.1.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/spf13/pflag v1.0.9 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/ysmood/fetchup v0.2.3 // indirect
|
||||
github.com/ysmood/goob v0.4.0 // indirect
|
||||
github.com/ysmood/got v0.40.0 // indirect
|
||||
github.com/ysmood/gson v0.7.3 // indirect
|
||||
github.com/ysmood/leakless v0.9.0 // indirect
|
||||
golang.org/x/sync v0.21.0 // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
golang.org/x/text v0.38.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,96 @@
|
||||
charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410 h1:D9PbaszZYpB4nj+d6HTWr1onlmlyuGVNfL9gAi8iB3k=
|
||||
charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410/go.mod h1:1qZyvvVCenJO2M1ac2mX0yyiIZJoZmDM4DG4s0udJkU=
|
||||
github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY=
|
||||
github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E=
|
||||
github.com/charmbracelet/colorprofile v0.3.3 h1:DjJzJtLP6/NZ8p7Cgjno0CKGr7wwRJGxWUwh2IyhfAI=
|
||||
github.com/charmbracelet/colorprofile v0.3.3/go.mod h1:nB1FugsAbzq284eJcjfah2nhdSLppN2NqvfotkfRYP4=
|
||||
github.com/charmbracelet/fang v1.0.0 h1:jESBY40agJOlLYnnv9jE0mLqDGTxEk0hkOnx7YGyRlQ=
|
||||
github.com/charmbracelet/fang v1.0.0/go.mod h1:P5/DNb9DddQ0Z0dbc0P3ol4/ix5Po7Ofr2KMBfAqoCo=
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20251106190538-99ea45596692 h1:r/3jQZ1LjWW6ybp8HHfhrKrwHIWiJhUuY7wwYIWZulQ=
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20251106190538-99ea45596692/go.mod h1:Y8B4DzWeTb0ama8l3+KyopZtkE8fZjwRQ3aEAPEXHE0=
|
||||
github.com/charmbracelet/x/ansi v0.11.0 h1:uuIVK7GIplwX6UBIz8S2TF8nkr7xRlygSsBRjSJqIvA=
|
||||
github.com/charmbracelet/x/ansi v0.11.0/go.mod h1:uQt8bOrq/xgXjlGcFMc8U2WYbnxyjrKhnvTQluvfCaE=
|
||||
github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444 h1:IJDiTgVE56gkAGfq0lBEloWgkXMk4hl/bmuPoicI4R0=
|
||||
github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444/go.mod h1:T9jr8CzFpjhFVHjNjKwbAD7KwBNyFnj2pntAO7F2zw0=
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f h1:pk6gmGpCE7F3FcjaOEKYriCvpmIN4+6OS/RD0vm4uIA=
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f/go.mod h1:IfZAMTHB6XkZSeXUqriemErjAWCCzT0LwjKFYCZyw0I=
|
||||
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
|
||||
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
|
||||
github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY=
|
||||
github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo=
|
||||
github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2/jYn2GuM=
|
||||
github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k=
|
||||
github.com/clipperhouse/displaywidth v0.4.1 h1:uVw9V8UDfnggg3K2U84VWY1YLQ/x2aKSCtkRyYozfoU=
|
||||
github.com/clipperhouse/displaywidth v0.4.1/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o=
|
||||
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
|
||||
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
|
||||
github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
|
||||
github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
|
||||
github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4=
|
||||
github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/mango v0.1.0 h1:DZQK45d2gGbql1arsYA4vfg4d7I9Hfx5rX/GCmzsAvI=
|
||||
github.com/muesli/mango v0.1.0/go.mod h1:5XFpbC8jY5UUv89YQciiXNlbi+iJgt29VDC5xbzrLL4=
|
||||
github.com/muesli/mango-cobra v1.2.0 h1:DQvjzAM0PMZr85Iv9LIMaYISpTOliMEg+uMFtNbYvWg=
|
||||
github.com/muesli/mango-cobra v1.2.0/go.mod h1:vMJL54QytZAJhCT13LPVDfkvCUJ5/4jNUKF/8NC2UjA=
|
||||
github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe7Sg=
|
||||
github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0=
|
||||
github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8=
|
||||
github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
|
||||
github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
|
||||
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
|
||||
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
|
||||
github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
|
||||
github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg=
|
||||
github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
|
||||
github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM=
|
||||
github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q=
|
||||
github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
|
||||
github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
|
||||
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
|
||||
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
|
||||
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
|
||||
github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
||||
github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
|
||||
github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,218 @@
|
||||
// Package robots is a small, pure robots.txt parser and matcher. kage obeys it
|
||||
// by default so a clone stays polite; --no-robots bypasses the matcher and puts
|
||||
// the consequences on the user.
|
||||
package robots
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// rule is one Allow or Disallow directive.
|
||||
type rule struct {
|
||||
pattern string
|
||||
allow bool
|
||||
}
|
||||
|
||||
// Matcher answers whether a path may be crawled for one user-agent group, and
|
||||
// carries the sitemaps and crawl-delay declared in the file.
|
||||
type Matcher struct {
|
||||
rules []rule
|
||||
Sitemaps []string
|
||||
CrawlDelay time.Duration
|
||||
}
|
||||
|
||||
// AllowAll returns a Matcher that permits every path (used when robots is
|
||||
// disabled or the file is absent).
|
||||
func AllowAll() *Matcher { return &Matcher{} }
|
||||
|
||||
// group accumulates the rules attached to a set of user-agent lines.
|
||||
type group struct {
|
||||
agents []string
|
||||
rules []rule
|
||||
delay time.Duration
|
||||
}
|
||||
|
||||
// Parse reads robots.txt content and returns a Matcher for the given agent
|
||||
// token (e.g. "kage"). The most specific matching group is used, falling back to
|
||||
// the "*" group. Sitemaps are collected globally.
|
||||
func Parse(data string, agent string) *Matcher {
|
||||
agent = strings.ToLower(agent)
|
||||
var groups []*group
|
||||
var cur *group
|
||||
var sitemaps []string
|
||||
startedRules := false
|
||||
|
||||
for line := range strings.SplitSeq(data, "\n") {
|
||||
line = stripComment(line)
|
||||
key, val, ok := splitField(line)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch strings.ToLower(key) {
|
||||
case "user-agent":
|
||||
// A user-agent line after rules starts a fresh group.
|
||||
if cur == nil || startedRules {
|
||||
cur = &group{}
|
||||
groups = append(groups, cur)
|
||||
startedRules = false
|
||||
}
|
||||
cur.agents = append(cur.agents, strings.ToLower(val))
|
||||
case "allow":
|
||||
if cur != nil {
|
||||
cur.rules = append(cur.rules, rule{pattern: val, allow: true})
|
||||
startedRules = true
|
||||
}
|
||||
case "disallow":
|
||||
if cur != nil {
|
||||
cur.rules = append(cur.rules, rule{pattern: val, allow: false})
|
||||
startedRules = true
|
||||
}
|
||||
case "crawl-delay":
|
||||
if cur != nil {
|
||||
if d := parseDelay(val); d > 0 {
|
||||
cur.delay = d
|
||||
}
|
||||
startedRules = true
|
||||
}
|
||||
case "sitemap":
|
||||
if val != "" {
|
||||
sitemaps = append(sitemaps, val)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m := &Matcher{Sitemaps: sitemaps}
|
||||
if g := selectGroup(groups, agent); g != nil {
|
||||
m.rules = g.rules
|
||||
m.CrawlDelay = g.delay
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// selectGroup picks the group whose agent best matches: an exact/substring
|
||||
// match on the agent token beats the wildcard "*" group.
|
||||
func selectGroup(groups []*group, agent string) *group {
|
||||
var star, specific *group
|
||||
bestLen := -1
|
||||
for _, g := range groups {
|
||||
for _, a := range g.agents {
|
||||
if a == "*" {
|
||||
star = g
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(agent, a) || strings.Contains(agent, a) {
|
||||
if len(a) > bestLen {
|
||||
bestLen = len(a)
|
||||
specific = g
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if specific != nil {
|
||||
return specific
|
||||
}
|
||||
return star
|
||||
}
|
||||
|
||||
// Allowed reports whether path may be crawled. The longest matching rule wins;
|
||||
// on a tie, Allow beats Disallow. An empty Disallow means "allow everything".
|
||||
func (m *Matcher) Allowed(path string) bool {
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
bestLen := -1
|
||||
bestAllow := true
|
||||
for _, r := range m.rules {
|
||||
if r.pattern == "" {
|
||||
// Empty Disallow = allow all; empty Allow is a no-op.
|
||||
continue
|
||||
}
|
||||
if matchPattern(r.pattern, path) {
|
||||
pl := len(r.pattern)
|
||||
if pl > bestLen || (pl == bestLen && r.allow) {
|
||||
bestLen = pl
|
||||
bestAllow = r.allow
|
||||
}
|
||||
}
|
||||
}
|
||||
if bestLen < 0 {
|
||||
return true
|
||||
}
|
||||
return bestAllow
|
||||
}
|
||||
|
||||
// matchPattern matches a robots path pattern against path, honouring '*'
|
||||
// (any run) and a trailing '$' (end anchor).
|
||||
func matchPattern(pattern, path string) bool {
|
||||
anchored := strings.HasSuffix(pattern, "$")
|
||||
if anchored {
|
||||
pattern = pattern[:len(pattern)-1]
|
||||
}
|
||||
segs := strings.Split(pattern, "*")
|
||||
pos := 0
|
||||
for i, seg := range segs {
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
if i == 0 {
|
||||
if !strings.HasPrefix(path[pos:], seg) {
|
||||
return false
|
||||
}
|
||||
pos += len(seg)
|
||||
continue
|
||||
}
|
||||
idx := strings.Index(path[pos:], seg)
|
||||
if idx < 0 {
|
||||
return false
|
||||
}
|
||||
pos += idx + len(seg)
|
||||
}
|
||||
if anchored {
|
||||
// The last non-empty segment must land exactly at the end.
|
||||
last := ""
|
||||
for i := len(segs) - 1; i >= 0; i-- {
|
||||
if segs[i] != "" {
|
||||
last = segs[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if last != "" && !strings.HasSuffix(path, last) {
|
||||
return false
|
||||
}
|
||||
// A pattern with no trailing wildcard must consume the whole path.
|
||||
if !strings.HasSuffix(pattern, "*") && pos != len(path) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func stripComment(line string) string {
|
||||
if before, _, found := strings.Cut(line, "#"); found {
|
||||
return before
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func splitField(line string) (key, val string, ok bool) {
|
||||
k, v, found := strings.Cut(line, ":")
|
||||
if !found {
|
||||
return "", "", false
|
||||
}
|
||||
key = strings.TrimSpace(k)
|
||||
val = strings.TrimSpace(v)
|
||||
if key == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return key, val, true
|
||||
}
|
||||
|
||||
func parseDelay(val string) time.Duration {
|
||||
secs, err := strconv.ParseFloat(strings.TrimSpace(val), 64)
|
||||
if err != nil || secs <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(secs * float64(time.Second))
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package robots
|
||||
|
||||
import "testing"
|
||||
|
||||
const sample = `
|
||||
# example robots
|
||||
User-agent: *
|
||||
Disallow: /private/
|
||||
Allow: /private/public
|
||||
Disallow: /*.json$
|
||||
Crawl-delay: 2
|
||||
|
||||
User-agent: kage
|
||||
Disallow: /no-kage/
|
||||
|
||||
Sitemap: https://ex.com/sitemap.xml
|
||||
Sitemap: https://ex.com/news.xml
|
||||
`
|
||||
|
||||
func TestParseAndAllowedSpecificAgent(t *testing.T) {
|
||||
m := Parse(sample, "kage")
|
||||
// kage's own group only disallows /no-kage/.
|
||||
if m.Allowed("/private/secret") == false {
|
||||
t.Error("kage group should allow /private/secret (only /no-kage/ is blocked)")
|
||||
}
|
||||
if m.Allowed("/no-kage/x") {
|
||||
t.Error("/no-kage/x should be disallowed for kage")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowedWildcardGroup(t *testing.T) {
|
||||
m := Parse(sample, "somebot")
|
||||
cases := map[string]bool{
|
||||
"/": true,
|
||||
"/private/x": false, // Disallow /private/
|
||||
"/private/public": true, // longer Allow wins on tie/length
|
||||
"/data/file.json": false, // Disallow /*.json$
|
||||
"/data/file.jsonx": true, // $ anchors the extension
|
||||
"/ok": true,
|
||||
}
|
||||
for p, want := range cases {
|
||||
if got := m.Allowed(p); got != want {
|
||||
t.Errorf("Allowed(%q) = %v, want %v", p, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSitemapsAndDelay(t *testing.T) {
|
||||
m := Parse(sample, "somebot")
|
||||
if len(m.Sitemaps) != 2 {
|
||||
t.Fatalf("got %d sitemaps, want 2", len(m.Sitemaps))
|
||||
}
|
||||
if m.Sitemaps[0] != "https://ex.com/sitemap.xml" {
|
||||
t.Errorf("sitemap[0] = %q", m.Sitemaps[0])
|
||||
}
|
||||
if m.CrawlDelay.Seconds() != 2 {
|
||||
t.Errorf("crawl-delay = %v, want 2s", m.CrawlDelay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyDisallowAllowsAll(t *testing.T) {
|
||||
m := Parse("User-agent: *\nDisallow:\n", "x")
|
||||
if !m.Allowed("/anything") {
|
||||
t.Error("empty Disallow should allow everything")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoRobotsAllowsAll(t *testing.T) {
|
||||
m := AllowAll()
|
||||
if !m.Allowed("/whatever") {
|
||||
t.Error("AllowAll must allow everything")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
// Package sanitize removes every trace of JavaScript from an HTML document so
|
||||
// the saved page is inert: a photograph, not a program.
|
||||
//
|
||||
// It parses with golang.org/x/net/html, walks the tree, and deletes scripts,
|
||||
// event handlers, javascript: URLs, and the dead preconnect/preload hints that
|
||||
// mean nothing offline — while leaving styles, images, fonts, forms, and all
|
||||
// semantic markup untouched so the layout survives intact.
|
||||
package sanitize
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/net/html/atom"
|
||||
)
|
||||
|
||||
// Options tune a few edge behaviours; the zero value is the safe default
|
||||
// (scripts and noscript removed, meta-refresh removed).
|
||||
type Options struct {
|
||||
// KeepNoscript unwraps <noscript> content into the document instead of
|
||||
// deleting it, for sites whose real content hides behind a JS check.
|
||||
KeepNoscript bool
|
||||
// KeepMetaRefresh preserves a plain timed <meta http-equiv="refresh">
|
||||
// (a JS-target refresh is always removed).
|
||||
KeepMetaRefresh bool
|
||||
// Banner, when non-empty, is inserted as an HTML comment at the top of the
|
||||
// document.
|
||||
Banner string
|
||||
}
|
||||
|
||||
// Report counts what was removed, for the run summary and for tests.
|
||||
type Report struct {
|
||||
ScriptsRemoved int
|
||||
HandlersRemoved int
|
||||
NoscriptRemoved int
|
||||
NoscriptUnwrapped int
|
||||
JSURLsNeutralized int
|
||||
MetaRefreshRemoved int
|
||||
DeadLinksRemoved int
|
||||
}
|
||||
|
||||
// jsURLAttrs are attributes whose value may be a javascript: URL.
|
||||
var jsURLAttrs = map[string]bool{
|
||||
"href": true, "src": true, "action": true, "formaction": true,
|
||||
"poster": true, "data": true, "background": true, "xlink:href": true,
|
||||
}
|
||||
|
||||
// Strip parses doc, removes all JavaScript, and returns the rewritten HTML plus
|
||||
// a Report. A parse error is returned unchanged to the caller.
|
||||
func Strip(doc []byte, opts Options) ([]byte, Report, error) {
|
||||
root, err := html.Parse(bytes.NewReader(doc))
|
||||
if err != nil {
|
||||
return nil, Report{}, err
|
||||
}
|
||||
rep := CleanTree(root, opts)
|
||||
var buf bytes.Buffer
|
||||
if err := html.Render(&buf, root); err != nil {
|
||||
return nil, rep, err
|
||||
}
|
||||
return buf.Bytes(), rep, nil
|
||||
}
|
||||
|
||||
// CleanTree removes all JavaScript from an already-parsed document in place and
|
||||
// returns the Report. The cloner uses this so the HTML is parsed only once and
|
||||
// shared with the asset rewriter.
|
||||
func CleanTree(root *html.Node, opts Options) Report {
|
||||
var rep Report
|
||||
clean(root, opts, &rep)
|
||||
if opts.Banner != "" {
|
||||
insertBanner(root, opts.Banner)
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
// clean walks n's children, removing or scrubbing each before recursing.
|
||||
func clean(n *html.Node, opts Options, rep *Report) {
|
||||
var next *html.Node
|
||||
for c := n.FirstChild; c != nil; c = next {
|
||||
next = c.NextSibling
|
||||
if c.Type == html.ElementNode {
|
||||
switch c.DataAtom {
|
||||
case atom.Script:
|
||||
n.RemoveChild(c)
|
||||
rep.ScriptsRemoved++
|
||||
continue
|
||||
case atom.Noscript:
|
||||
if opts.KeepNoscript {
|
||||
unwrapNoscript(n, c)
|
||||
rep.NoscriptUnwrapped++
|
||||
} else {
|
||||
n.RemoveChild(c)
|
||||
rep.NoscriptRemoved++
|
||||
}
|
||||
continue
|
||||
case atom.Meta:
|
||||
if isMetaRefresh(c) && (!opts.KeepMetaRefresh || isJSRefresh(c)) {
|
||||
n.RemoveChild(c)
|
||||
rep.MetaRefreshRemoved++
|
||||
continue
|
||||
}
|
||||
case atom.Link:
|
||||
if isDeadLink(c) {
|
||||
n.RemoveChild(c)
|
||||
rep.DeadLinksRemoved++
|
||||
continue
|
||||
}
|
||||
}
|
||||
stripHandlers(c, rep)
|
||||
neutralizeJSURLs(c, rep)
|
||||
}
|
||||
clean(c, opts, rep)
|
||||
}
|
||||
}
|
||||
|
||||
// stripHandlers removes every on* event-handler attribute from n.
|
||||
func stripHandlers(n *html.Node, rep *Report) {
|
||||
kept := n.Attr[:0]
|
||||
for _, a := range n.Attr {
|
||||
if len(a.Key) > 2 && strings.HasPrefix(strings.ToLower(a.Key), "on") {
|
||||
rep.HandlersRemoved++
|
||||
continue
|
||||
}
|
||||
kept = append(kept, a)
|
||||
}
|
||||
n.Attr = kept
|
||||
}
|
||||
|
||||
// neutralizeJSURLs replaces javascript: URLs: links become "#", other carriers
|
||||
// lose the attribute entirely.
|
||||
func neutralizeJSURLs(n *html.Node, rep *Report) {
|
||||
kept := n.Attr[:0]
|
||||
for _, a := range n.Attr {
|
||||
key := strings.ToLower(a.Key)
|
||||
if jsURLAttrs[key] && strings.HasPrefix(strings.ToLower(strings.TrimSpace(a.Val)), "javascript:") {
|
||||
rep.JSURLsNeutralized++
|
||||
if key == "href" {
|
||||
a.Val = "#"
|
||||
kept = append(kept, a)
|
||||
}
|
||||
// non-href carriers: drop the attribute.
|
||||
continue
|
||||
}
|
||||
kept = append(kept, a)
|
||||
}
|
||||
n.Attr = kept
|
||||
}
|
||||
|
||||
// isMetaRefresh reports whether n is a <meta http-equiv="refresh">.
|
||||
func isMetaRefresh(n *html.Node) bool {
|
||||
return strings.EqualFold(attr(n, "http-equiv"), "refresh")
|
||||
}
|
||||
|
||||
// isJSRefresh reports whether a meta-refresh target is a javascript: URL.
|
||||
func isJSRefresh(n *html.Node) bool {
|
||||
return strings.Contains(strings.ToLower(attr(n, "content")), "javascript:")
|
||||
}
|
||||
|
||||
// isDeadLink reports whether a <link> is a resource hint that is useless or
|
||||
// script-bound offline: preconnect, dns-prefetch, modulepreload, or a
|
||||
// preload/prefetch that targets a script.
|
||||
func isDeadLink(n *html.Node) bool {
|
||||
for r := range strings.FieldsSeq(strings.ToLower(attr(n, "rel"))) {
|
||||
switch r {
|
||||
case "preconnect", "dns-prefetch", "modulepreload":
|
||||
return true
|
||||
case "preload", "prefetch":
|
||||
as := strings.ToLower(attr(n, "as"))
|
||||
href := strings.ToLower(attr(n, "href"))
|
||||
if as == "script" || strings.HasSuffix(href, ".js") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// unwrapNoscript replaces a <noscript> with its content. Because x/net/html
|
||||
// parses noscript content as raw text (scripting enabled), the text is
|
||||
// re-parsed as a fragment in the parent's context and spliced in before the
|
||||
// noscript node, which is then removed.
|
||||
func unwrapNoscript(parent, ns *html.Node) {
|
||||
var raw strings.Builder
|
||||
for c := ns.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type == html.TextNode {
|
||||
raw.WriteString(c.Data)
|
||||
}
|
||||
}
|
||||
frag, err := html.ParseFragment(strings.NewReader(raw.String()), &html.Node{
|
||||
Type: html.ElementNode,
|
||||
Data: "body",
|
||||
DataAtom: atom.Body,
|
||||
})
|
||||
if err == nil {
|
||||
for _, fn := range frag {
|
||||
parent.InsertBefore(fn, ns)
|
||||
}
|
||||
}
|
||||
parent.RemoveChild(ns)
|
||||
}
|
||||
|
||||
// insertBanner prepends an HTML comment to the document.
|
||||
func insertBanner(root *html.Node, text string) {
|
||||
c := &html.Node{Type: html.CommentNode, Data: " " + text + " "}
|
||||
if root.FirstChild != nil {
|
||||
root.InsertBefore(c, root.FirstChild)
|
||||
} else {
|
||||
root.AppendChild(c)
|
||||
}
|
||||
}
|
||||
|
||||
// attr returns the value of n's attribute key (case-insensitive), or "".
|
||||
func attr(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if strings.EqualFold(a.Key, key) {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package sanitize
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const page = `<!doctype html>
|
||||
<html><head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="refresh" content="5;url=https://ex.com/next">
|
||||
<title>Hi</title>
|
||||
<link rel="stylesheet" href="/css/main.css">
|
||||
<link rel="preconnect" href="https://cdn.io">
|
||||
<link rel="modulepreload" href="/app.js">
|
||||
<link rel="preload" as="script" href="/runtime.js">
|
||||
<style>.a{color:red}</style>
|
||||
<script src="/vendor.js"></script>
|
||||
<script>window.x=1</script>
|
||||
</head>
|
||||
<body onload="boot()">
|
||||
<h1 onclick="go()">Title</h1>
|
||||
<a href="javascript:open()">js link</a>
|
||||
<a href="/real">real link</a>
|
||||
<img src="/logo.png" onerror="fail()">
|
||||
<form action="/submit"><input name="q"></form>
|
||||
<noscript><p>need js</p></noscript>
|
||||
<p style="background:url(/bg.png)">styled</p>
|
||||
</body></html>`
|
||||
|
||||
func TestStripRemovesAllJS(t *testing.T) {
|
||||
out, rep, err := Strip([]byte(page), Options{Banner: "cloned by kage"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(out)
|
||||
|
||||
if strings.Contains(s, "<script") {
|
||||
t.Error("a <script> survived")
|
||||
}
|
||||
if strings.Contains(s, "onload") || strings.Contains(s, "onclick") || strings.Contains(s, "onerror") {
|
||||
t.Error("an on* handler survived")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(s), "javascript:") {
|
||||
t.Error("a javascript: URL survived")
|
||||
}
|
||||
if strings.Contains(s, "modulepreload") || strings.Contains(s, "preconnect") {
|
||||
t.Error("a dead resource hint survived")
|
||||
}
|
||||
if strings.Contains(s, "http-equiv") {
|
||||
t.Error("a meta refresh survived")
|
||||
}
|
||||
|
||||
// Layout-bearing markup must survive untouched.
|
||||
for _, want := range []string{
|
||||
`rel="stylesheet"`, `href="/css/main.css"`,
|
||||
`<style>`, `color:red`,
|
||||
`src="/logo.png"`, `<form action="/submit">`,
|
||||
`background:url(/bg.png)`, `href="/real"`,
|
||||
"<!-- cloned by kage -->",
|
||||
} {
|
||||
if !strings.Contains(s, want) {
|
||||
t.Errorf("expected %q to survive, output:\n%s", want, s)
|
||||
}
|
||||
}
|
||||
|
||||
// The js link keeps an anchor but points nowhere dangerous.
|
||||
if !strings.Contains(s, `href="#"`) {
|
||||
t.Error("javascript: link should be neutralized to href=#")
|
||||
}
|
||||
|
||||
if rep.ScriptsRemoved != 2 {
|
||||
t.Errorf("ScriptsRemoved = %d, want 2", rep.ScriptsRemoved)
|
||||
}
|
||||
if rep.HandlersRemoved != 3 {
|
||||
t.Errorf("HandlersRemoved = %d, want 3", rep.HandlersRemoved)
|
||||
}
|
||||
if rep.JSURLsNeutralized != 1 {
|
||||
t.Errorf("JSURLsNeutralized = %d, want 1", rep.JSURLsNeutralized)
|
||||
}
|
||||
if rep.MetaRefreshRemoved != 1 {
|
||||
t.Errorf("MetaRefreshRemoved = %d, want 1", rep.MetaRefreshRemoved)
|
||||
}
|
||||
if rep.DeadLinksRemoved != 3 {
|
||||
t.Errorf("DeadLinksRemoved = %d, want 3", rep.DeadLinksRemoved)
|
||||
}
|
||||
if rep.NoscriptRemoved != 1 {
|
||||
t.Errorf("NoscriptRemoved = %d, want 1", rep.NoscriptRemoved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeepNoscriptUnwraps(t *testing.T) {
|
||||
in := `<html><body><noscript><p>fallback text</p></noscript></body></html>`
|
||||
out, rep, err := Strip([]byte(in), Options{KeepNoscript: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(out)
|
||||
if strings.Contains(s, "<noscript") {
|
||||
t.Error("noscript wrapper should be gone")
|
||||
}
|
||||
if !strings.Contains(s, "fallback text") {
|
||||
t.Errorf("unwrapped content missing: %s", s)
|
||||
}
|
||||
if rep.NoscriptUnwrapped != 1 {
|
||||
t.Errorf("NoscriptUnwrapped = %d, want 1", rep.NoscriptUnwrapped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeepMetaRefreshPlain(t *testing.T) {
|
||||
in := `<html><head><meta http-equiv="refresh" content="5;url=/next"></head><body></body></html>`
|
||||
out, _, err := Strip([]byte(in), Options{KeepMetaRefresh: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(out), "http-equiv") {
|
||||
t.Error("plain meta refresh should be kept when KeepMetaRefresh is set")
|
||||
}
|
||||
|
||||
// A JS-target refresh is removed even when KeepMetaRefresh is set.
|
||||
js := `<html><head><meta http-equiv="refresh" content="0;url=javascript:alert(1)"></head><body></body></html>`
|
||||
out2, _, _ := Strip([]byte(js), Options{KeepMetaRefresh: true})
|
||||
if strings.Contains(strings.ToLower(string(out2)), "javascript:") {
|
||||
t.Error("JS-target meta refresh must be removed regardless")
|
||||
}
|
||||
}
|
||||
Executable
+144
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Idempotently configure a Cloudflare Pages project, custom domain, and DNS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
API = "https://api.cloudflare.com/client/v4"
|
||||
|
||||
|
||||
def request(method: str, url: str, token: str, payload: dict[str, object] | None = None) -> dict[str, object]:
|
||||
data = None if payload is None else json.dumps(payload).encode()
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
method=method,
|
||||
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
body = resp.read().decode()
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read().decode()
|
||||
try:
|
||||
parsed = json.loads(body) if body else {}
|
||||
except json.JSONDecodeError:
|
||||
parsed = {"raw": body}
|
||||
raise SystemExit(f"{method} {url} failed: HTTP {exc.code} {parsed}") from exc
|
||||
|
||||
parsed = json.loads(body) if body else {}
|
||||
if not parsed.get("success", True):
|
||||
raise SystemExit(f"{method} {url} failed: {parsed}")
|
||||
return parsed
|
||||
|
||||
|
||||
def ensure_project(account_id: str, token: str, project: str, branch: str) -> str:
|
||||
url = f"{API}/accounts/{account_id}/pages/projects/{project}"
|
||||
try:
|
||||
result = request("GET", url, token)
|
||||
print(f"project exists: {project}")
|
||||
return result["result"]["subdomain"]
|
||||
except SystemExit as exc:
|
||||
if "HTTP 404" not in str(exc):
|
||||
raise
|
||||
|
||||
result = request(
|
||||
"POST",
|
||||
f"{API}/accounts/{account_id}/pages/projects",
|
||||
token,
|
||||
{"name": project, "production_branch": branch},
|
||||
)
|
||||
print(f"created project: {project}")
|
||||
return result["result"]["subdomain"]
|
||||
|
||||
|
||||
def ensure_custom_domain(account_id: str, token: str, project: str, domain: str) -> None:
|
||||
base = f"{API}/accounts/{account_id}/pages/projects/{project}/domains"
|
||||
try:
|
||||
request("GET", f"{base}/{domain}", token)
|
||||
print(f"domain exists: {domain}")
|
||||
return
|
||||
except SystemExit as exc:
|
||||
if "HTTP 404" not in str(exc):
|
||||
raise
|
||||
|
||||
request("POST", base, token, {"name": domain})
|
||||
print(f"attached domain: {domain}")
|
||||
|
||||
|
||||
def find_zone(token: str, zone_name: str) -> str:
|
||||
query = urllib.parse.urlencode({"name": zone_name})
|
||||
result = request("GET", f"{API}/zones?{query}", token)
|
||||
zones = result.get("result") or []
|
||||
if not zones:
|
||||
raise SystemExit(f"zone not found: {zone_name}")
|
||||
return zones[0]["id"]
|
||||
|
||||
|
||||
def ensure_dns(token: str, zone_name: str, domain: str, target: str) -> None:
|
||||
zone_id = find_zone(token, zone_name)
|
||||
base = f"{API}/zones/{zone_id}/dns_records"
|
||||
query = urllib.parse.urlencode({"name": domain})
|
||||
result = request("GET", f"{base}?{query}", token)
|
||||
payload = {"type": "CNAME", "name": domain, "content": target, "ttl": 1, "proxied": True}
|
||||
matches = result.get("result") or []
|
||||
|
||||
if matches:
|
||||
record_id = matches[0]["id"]
|
||||
record = matches[0]
|
||||
if (
|
||||
record.get("type") == payload["type"]
|
||||
and record.get("content") == payload["content"]
|
||||
and record.get("proxied") is True
|
||||
):
|
||||
print(f"DNS exists: {domain} -> {target}")
|
||||
return
|
||||
request("PUT", f"{base}/{record_id}", token, payload)
|
||||
print(f"updated DNS: {domain} -> {target}")
|
||||
return
|
||||
|
||||
request("POST", base, token, payload)
|
||||
print(f"created DNS: {domain} -> {target}")
|
||||
|
||||
|
||||
def default_zone(domain: str) -> str:
|
||||
parts = domain.split(".")
|
||||
if len(parts) < 2:
|
||||
raise SystemExit(f"cannot infer zone from domain: {domain}")
|
||||
return ".".join(parts[-2:])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--project", required=True)
|
||||
parser.add_argument("--domain")
|
||||
parser.add_argument("--zone")
|
||||
parser.add_argument("--branch", default="main")
|
||||
parser.add_argument("--dns-target")
|
||||
args = parser.parse_args()
|
||||
|
||||
account_id = os.environ["CLOUDFLARE_ACCOUNT_ID"]
|
||||
token = os.environ["CLOUDFLARE_API_TOKEN"]
|
||||
|
||||
# The pages.dev subdomain is not always {project}.pages.dev: when the
|
||||
# name is taken globally, Cloudflare assigns a suffixed one. A CNAME at
|
||||
# the guessed name points at a stranger's project and the custom domain
|
||||
# never validates, so always ask the API for the real subdomain.
|
||||
subdomain = ensure_project(account_id, token, args.project, args.branch)
|
||||
if args.domain:
|
||||
target = args.dns_target or subdomain
|
||||
zone = args.zone or default_zone(args.domain)
|
||||
ensure_custom_domain(account_id, token, args.project, args.domain)
|
||||
ensure_dns(token, zone, args.domain, target)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+318
@@ -0,0 +1,318 @@
|
||||
// Package urlx is the URL ⇄ filesystem contract at the heart of kage.
|
||||
//
|
||||
// Every reference kage meets — a page link, a stylesheet, an image, a font —
|
||||
// is funnelled through Normalize so that two different-looking URLs that point
|
||||
// at the same resource collapse to one canonical key. LocalPath then maps that
|
||||
// canonical URL to a deterministic path on disk, and Rel turns two such paths
|
||||
// into the relative link that goes back into the rewritten HTML or CSS.
|
||||
//
|
||||
// The package is pure: no network, no filesystem, no clock. That is what makes
|
||||
// the rest of kage easy to reason about — a page worker can rewrite a link to
|
||||
// an asset long before the asset has been downloaded, because both sides agree
|
||||
// on where the bytes will live.
|
||||
package urlx
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Kind distinguishes a crawlable page from a downloadable asset; the two map to
|
||||
// different places on disk (pages mirror the URL path, assets live under the
|
||||
// reserved prefix).
|
||||
type Kind int
|
||||
|
||||
const (
|
||||
// Page is an HTML document kage renders and rewrites.
|
||||
Page Kind = iota
|
||||
// Asset is a stylesheet, image, font, or media file kage downloads verbatim.
|
||||
Asset
|
||||
)
|
||||
|
||||
// DefaultReserved is the directory under the mirror root where every asset and
|
||||
// kage's own state live. It is deliberately unlikely to collide with a real URL
|
||||
// path segment.
|
||||
const DefaultReserved = "_kage"
|
||||
|
||||
// binaryExts are extensions that, when seen on an <a href>, mean the link points
|
||||
// at a file to download rather than a page to render.
|
||||
var binaryExts = map[string]bool{
|
||||
".pdf": true, ".zip": true, ".gz": true, ".tar": true, ".tgz": true,
|
||||
".rar": true, ".7z": true, ".dmg": true, ".exe": true, ".msi": true,
|
||||
".png": true, ".jpg": true, ".jpeg": true, ".gif": true, ".webp": true,
|
||||
".svg": true, ".ico": true, ".bmp": true, ".avif": true,
|
||||
".mp3": true, ".mp4": true, ".webm": true, ".mov": true, ".avi": true,
|
||||
".wav": true, ".ogg": true, ".flac": true,
|
||||
".woff": true, ".woff2": true, ".ttf": true, ".otf": true, ".eot": true,
|
||||
".css": true, ".js": true, ".json": true, ".xml": true, ".rss": true,
|
||||
".doc": true, ".docx": true, ".xls": true, ".xlsx": true, ".ppt": true,
|
||||
".pptx": true, ".csv": true, ".txt": true,
|
||||
}
|
||||
|
||||
// ParseSeed turns a command-line argument like "example.com",
|
||||
// "https://example.com/docs" or "http://ex.com" into a canonical absolute URL.
|
||||
// A bare host (no scheme) is assumed to be https.
|
||||
func ParseSeed(arg string) (*url.URL, error) {
|
||||
s := strings.TrimSpace(arg)
|
||||
if s == "" {
|
||||
return nil, fmt.Errorf("empty seed")
|
||||
}
|
||||
if !strings.Contains(s, "://") {
|
||||
s = "https://" + s
|
||||
}
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse seed %q: %w", arg, err)
|
||||
}
|
||||
if u.Host == "" {
|
||||
return nil, fmt.Errorf("seed %q has no host", arg)
|
||||
}
|
||||
return canonical(u, true), nil
|
||||
}
|
||||
|
||||
// Normalize resolves ref against base and canonicalises the result. It returns
|
||||
// an error for references kage cannot crawl or download: empty, fragment-only,
|
||||
// or a non-http(s) scheme (mailto:, tel:, data:, javascript:, blob:, …).
|
||||
func Normalize(base *url.URL, ref string) (*url.URL, error) {
|
||||
ref = strings.TrimSpace(ref)
|
||||
if ref == "" {
|
||||
return nil, fmt.Errorf("empty reference")
|
||||
}
|
||||
if strings.HasPrefix(ref, "#") {
|
||||
return nil, fmt.Errorf("fragment-only reference")
|
||||
}
|
||||
low := strings.ToLower(ref)
|
||||
for _, bad := range []string{"javascript:", "mailto:", "tel:", "data:", "blob:", "about:", "ftp:", "sms:"} {
|
||||
if strings.HasPrefix(low, bad) {
|
||||
return nil, fmt.Errorf("non-crawlable scheme: %s", bad)
|
||||
}
|
||||
}
|
||||
r, err := url.Parse(ref)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse ref %q: %w", ref, err)
|
||||
}
|
||||
var abs *url.URL
|
||||
if base != nil {
|
||||
abs = base.ResolveReference(r)
|
||||
} else {
|
||||
abs = r
|
||||
}
|
||||
if abs.Scheme != "http" && abs.Scheme != "https" {
|
||||
return nil, fmt.Errorf("non-http scheme: %q", abs.Scheme)
|
||||
}
|
||||
if abs.Host == "" {
|
||||
return nil, fmt.Errorf("reference has no host")
|
||||
}
|
||||
return canonical(abs, false), nil
|
||||
}
|
||||
|
||||
// canonical lower-cases scheme/host, drops the fragment and any default port,
|
||||
// and cleans the path while preserving a meaningful trailing slash. When root
|
||||
// is true an empty path becomes "/".
|
||||
func canonical(u *url.URL, root bool) *url.URL {
|
||||
c := *u
|
||||
c.Scheme = strings.ToLower(c.Scheme)
|
||||
c.Host = strings.ToLower(c.Host)
|
||||
c.Fragment = ""
|
||||
// Strip a default port.
|
||||
if h := c.Hostname(); h != "" {
|
||||
switch {
|
||||
case c.Scheme == "http" && c.Port() == "80":
|
||||
c.Host = h
|
||||
case c.Scheme == "https" && c.Port() == "443":
|
||||
c.Host = h
|
||||
}
|
||||
}
|
||||
if c.Path == "" {
|
||||
if root {
|
||||
c.Path = "/"
|
||||
}
|
||||
} else {
|
||||
trailing := strings.HasSuffix(c.Path, "/")
|
||||
cleaned := path.Clean(c.Path)
|
||||
if cleaned == "." {
|
||||
cleaned = "/"
|
||||
}
|
||||
if trailing && cleaned != "/" {
|
||||
cleaned += "/"
|
||||
}
|
||||
c.Path = cleaned
|
||||
}
|
||||
return &c
|
||||
}
|
||||
|
||||
// Key is the canonical string form used to dedup pages and assets.
|
||||
func Key(u *url.URL) string { return u.String() }
|
||||
|
||||
// ScopeConfig controls which page URLs are in scope for crawling.
|
||||
type ScopeConfig struct {
|
||||
IncludeSubdomains bool
|
||||
ScopePrefix string // only crawl paths under this prefix, e.g. "/docs/"
|
||||
ExcludePaths []string // skip any path containing one of these substrings
|
||||
}
|
||||
|
||||
// SameSite reports whether u belongs to the seed's site: the same host, or a
|
||||
// subdomain of it when allowSub is set.
|
||||
func SameSite(seed, u *url.URL, allowSub bool) bool {
|
||||
sh, uh := seed.Hostname(), u.Hostname()
|
||||
if sh == uh {
|
||||
return true
|
||||
}
|
||||
if allowSub && strings.HasSuffix(uh, "."+sh) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// InScope reports whether u should be crawled as a page given the seed and cfg.
|
||||
func InScope(seed, u *url.URL, cfg ScopeConfig) bool {
|
||||
if !SameSite(seed, u, cfg.IncludeSubdomains) {
|
||||
return false
|
||||
}
|
||||
if cfg.ScopePrefix != "" && !strings.HasPrefix(u.Path, cfg.ScopePrefix) {
|
||||
return false
|
||||
}
|
||||
for _, ex := range cfg.ExcludePaths {
|
||||
if ex != "" && strings.Contains(u.Path, ex) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// LikelyPage reports whether an <a href> target should be rendered as a page
|
||||
// rather than downloaded as a file. Links ending in a known binary/document
|
||||
// extension are treated as assets.
|
||||
func LikelyPage(u *url.URL) bool {
|
||||
ext := strings.ToLower(path.Ext(lastSegment(u.Path)))
|
||||
return !binaryExts[ext]
|
||||
}
|
||||
|
||||
// LocalPath maps a canonical URL to a slash-separated path relative to the
|
||||
// mirror root (out/<seedHost>/). Pages mirror the URL path as a directory index;
|
||||
// assets live under reserved/<host>/<path>. See spec §4.3.
|
||||
func LocalPath(seedHost string, u *url.URL, kind Kind, reserved string) string {
|
||||
if reserved == "" {
|
||||
reserved = DefaultReserved
|
||||
}
|
||||
host := u.Hostname()
|
||||
switch kind {
|
||||
case Asset:
|
||||
dir, base := splitAsset(u)
|
||||
base = applyQuery(base, u)
|
||||
return joinClean(reserved, host, dir, base)
|
||||
default: // Page
|
||||
dir := strings.Trim(u.Path, "/")
|
||||
leaf := applyQuery("index.html", u)
|
||||
if strings.EqualFold(host, seedHost) {
|
||||
return joinClean(dir, leaf)
|
||||
}
|
||||
// Subdomain pages get a host segment so they never collide with the
|
||||
// seed host's tree.
|
||||
return joinClean(host, dir, leaf)
|
||||
}
|
||||
}
|
||||
|
||||
// splitAsset breaks an asset URL path into a directory and a filename, inventing
|
||||
// a filename for directory-like or empty paths.
|
||||
func splitAsset(u *url.URL) (dir, base string) {
|
||||
clean := strings.Trim(u.Path, "/")
|
||||
switch {
|
||||
case clean == "":
|
||||
return "", "index"
|
||||
case strings.HasSuffix(u.Path, "/"):
|
||||
return clean, "index"
|
||||
default:
|
||||
if i := strings.LastIndex(clean, "/"); i >= 0 {
|
||||
return clean[:i], clean[i+1:]
|
||||
}
|
||||
return "", clean
|
||||
}
|
||||
}
|
||||
|
||||
// applyQuery folds a URL's query string into a filename as "__q-<hash>",
|
||||
// inserting it before the extension so the file keeps a sensible suffix.
|
||||
func applyQuery(name string, u *url.URL) string {
|
||||
if u.RawQuery == "" {
|
||||
return name
|
||||
}
|
||||
sum := sha256.Sum256([]byte(u.RawQuery))
|
||||
suffix := "__q-" + hex.EncodeToString(sum[:])[:6]
|
||||
if dot := strings.LastIndex(name, "."); dot > 0 {
|
||||
return name[:dot] + suffix + name[dot:]
|
||||
}
|
||||
return name + suffix
|
||||
}
|
||||
|
||||
// Rel returns the relative link from the directory of fromFile to toFile, both
|
||||
// slash paths relative to the mirror root. The result always uses '/'.
|
||||
func Rel(fromDir, toFile string) string {
|
||||
fromDir = path.Clean("/" + strings.TrimPrefix(fromDir, "/"))
|
||||
toFile = path.Clean("/" + strings.TrimPrefix(toFile, "/"))
|
||||
rel := relPath(fromDir, toFile)
|
||||
if rel == "" {
|
||||
return "."
|
||||
}
|
||||
return rel
|
||||
}
|
||||
|
||||
// relPath computes a relative path between two cleaned absolute slash paths,
|
||||
// treating 'from' as a directory.
|
||||
func relPath(fromDir, toFile string) string {
|
||||
from := splitNonEmpty(fromDir)
|
||||
to := splitNonEmpty(toFile)
|
||||
// Common prefix.
|
||||
i := 0
|
||||
for i < len(from) && i < len(to) && from[i] == to[i] {
|
||||
i++
|
||||
}
|
||||
var out []string
|
||||
for range from[i:] {
|
||||
out = append(out, "..")
|
||||
}
|
||||
out = append(out, to[i:]...)
|
||||
return strings.Join(out, "/")
|
||||
}
|
||||
|
||||
func splitNonEmpty(p string) []string {
|
||||
var out []string
|
||||
for s := range strings.SplitSeq(p, "/") {
|
||||
if s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Dir returns the directory portion of a slash file path.
|
||||
func Dir(file string) string {
|
||||
if i := strings.LastIndex(file, "/"); i >= 0 {
|
||||
return file[:i]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// joinClean joins path elements with '/', dropping empties and cleaning the
|
||||
// result, and never returns a leading slash.
|
||||
func joinClean(elems ...string) string {
|
||||
var parts []string
|
||||
for _, e := range elems {
|
||||
e = strings.Trim(e, "/")
|
||||
if e != "" {
|
||||
parts = append(parts, e)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func lastSegment(p string) string {
|
||||
p = strings.TrimRight(p, "/")
|
||||
if i := strings.LastIndex(p, "/"); i >= 0 {
|
||||
return p[i+1:]
|
||||
}
|
||||
return p
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
package urlx
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func mustParse(t *testing.T, s string) *url.URL {
|
||||
t.Helper()
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
t.Fatalf("parse %q: %v", s, err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
func TestParseSeed(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
err bool
|
||||
}{
|
||||
{"example.com", "https://example.com/", false},
|
||||
{"https://example.com", "https://example.com/", false},
|
||||
{"http://ex.com/docs", "http://ex.com/docs", false},
|
||||
{"HTTPS://Example.COM/A", "https://example.com/A", false},
|
||||
{"https://ex.com:443/", "https://ex.com/", false},
|
||||
{"http://ex.com:80/", "http://ex.com/", false},
|
||||
{"", "", true},
|
||||
{"http://", "", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := ParseSeed(c.in)
|
||||
if c.err {
|
||||
if err == nil {
|
||||
t.Errorf("ParseSeed(%q) want error, got %v", c.in, got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("ParseSeed(%q) unexpected error: %v", c.in, err)
|
||||
continue
|
||||
}
|
||||
if got.String() != c.want {
|
||||
t.Errorf("ParseSeed(%q) = %q, want %q", c.in, got.String(), c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalize(t *testing.T) {
|
||||
base := mustParse(t, "https://ex.com/docs/intro")
|
||||
cases := []struct {
|
||||
ref string
|
||||
want string
|
||||
err bool
|
||||
}{
|
||||
{"../guide", "https://ex.com/guide", false},
|
||||
{"/a/b/../c", "https://ex.com/a/c", false},
|
||||
{"page#frag", "https://ex.com/docs/page", false},
|
||||
{"//cdn.io/x.css", "https://cdn.io/x.css", false},
|
||||
{"HTTPS://Other.COM/Y", "https://other.com/Y", false},
|
||||
{"sub/", "https://ex.com/docs/sub/", false},
|
||||
{"#top", "", true},
|
||||
{"javascript:void(0)", "", true},
|
||||
{"mailto:a@b.com", "", true},
|
||||
{"data:image/png;base64,xxx", "", true},
|
||||
{"tel:+123", "", true},
|
||||
{"", "", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := Normalize(base, c.ref)
|
||||
if c.err {
|
||||
if err == nil {
|
||||
t.Errorf("Normalize(%q) want error, got %v", c.ref, got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("Normalize(%q) unexpected error: %v", c.ref, err)
|
||||
continue
|
||||
}
|
||||
if got.String() != c.want {
|
||||
t.Errorf("Normalize(%q) = %q, want %q", c.ref, got.String(), c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInScope(t *testing.T) {
|
||||
seed := mustParse(t, "https://ex.com/")
|
||||
cases := []struct {
|
||||
u string
|
||||
cfg ScopeConfig
|
||||
want bool
|
||||
}{
|
||||
{"https://ex.com/a", ScopeConfig{}, true},
|
||||
{"https://other.com/a", ScopeConfig{}, false},
|
||||
{"https://sub.ex.com/a", ScopeConfig{}, false},
|
||||
{"https://sub.ex.com/a", ScopeConfig{IncludeSubdomains: true}, true},
|
||||
{"https://ex.com/docs/x", ScopeConfig{ScopePrefix: "/docs/"}, true},
|
||||
{"https://ex.com/blog/x", ScopeConfig{ScopePrefix: "/docs/"}, false},
|
||||
{"https://ex.com/a/private/x", ScopeConfig{ExcludePaths: []string{"/private/"}}, false},
|
||||
{"https://ex.com/a/public/x", ScopeConfig{ExcludePaths: []string{"/private/"}}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := InScope(seed, mustParse(t, c.u), c.cfg)
|
||||
if got != c.want {
|
||||
t.Errorf("InScope(%q, %+v) = %v, want %v", c.u, c.cfg, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLikelyPage(t *testing.T) {
|
||||
cases := map[string]bool{
|
||||
"https://ex.com/docs": true,
|
||||
"https://ex.com/docs/": true,
|
||||
"https://ex.com/a.html": true,
|
||||
"https://ex.com/file.pdf": false,
|
||||
"https://ex.com/img.png": false,
|
||||
"https://ex.com/style.css": false,
|
||||
"https://ex.com/app.js": false,
|
||||
}
|
||||
for u, want := range cases {
|
||||
if got := LikelyPage(mustParse(t, u)); got != want {
|
||||
t.Errorf("LikelyPage(%q) = %v, want %v", u, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalPathPages(t *testing.T) {
|
||||
seed := "ex.com"
|
||||
cases := []struct {
|
||||
u string
|
||||
want string
|
||||
}{
|
||||
{"https://ex.com/", "index.html"},
|
||||
{"https://ex.com/docs", "docs/index.html"},
|
||||
{"https://ex.com/docs/", "docs/index.html"},
|
||||
{"https://ex.com/docs/intro", "docs/intro/index.html"},
|
||||
{"https://ex.com/a.html", "a.html/index.html"},
|
||||
{"https://sub.ex.com/x", "sub.ex.com/x/index.html"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := LocalPath(seed, mustParse(t, c.u), Page, "")
|
||||
if got != c.want {
|
||||
t.Errorf("LocalPath(page %q) = %q, want %q", c.u, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalPathPageQuery(t *testing.T) {
|
||||
got := LocalPath("ex.com", mustParse(t, "https://ex.com/search?q=cats"), Page, "")
|
||||
if !strings.HasPrefix(got, "search/index__q-") || !strings.HasSuffix(got, ".html") {
|
||||
t.Errorf("query page mapped to %q", got)
|
||||
}
|
||||
// Stable hash.
|
||||
got2 := LocalPath("ex.com", mustParse(t, "https://ex.com/search?q=cats"), Page, "")
|
||||
if got != got2 {
|
||||
t.Errorf("query hash not stable: %q vs %q", got, got2)
|
||||
}
|
||||
// Different query → different file.
|
||||
got3 := LocalPath("ex.com", mustParse(t, "https://ex.com/search?q=dogs"), Page, "")
|
||||
if got == got3 {
|
||||
t.Errorf("different queries collided: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalPathAssets(t *testing.T) {
|
||||
seed := "ex.com"
|
||||
cases := []struct {
|
||||
u string
|
||||
want string
|
||||
}{
|
||||
{"https://ex.com/css/main.css", "_kage/ex.com/css/main.css"},
|
||||
{"https://cdn.io/font/x.woff2", "_kage/cdn.io/font/x.woff2"},
|
||||
{"https://ex.com/avatar", "_kage/ex.com/avatar"},
|
||||
{"https://ex.com/assets/", "_kage/ex.com/assets/index"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := LocalPath(seed, mustParse(t, c.u), Asset, "")
|
||||
if got != c.want {
|
||||
t.Errorf("LocalPath(asset %q) = %q, want %q", c.u, got, c.want)
|
||||
}
|
||||
}
|
||||
// Query on an asset folds into the filename before the extension.
|
||||
got := LocalPath(seed, mustParse(t, "https://ex.com/img/logo.png?v=3"), Asset, "")
|
||||
if !strings.HasPrefix(got, "_kage/ex.com/img/logo__q-") || !strings.HasSuffix(got, ".png") {
|
||||
t.Errorf("asset query mapped to %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRel(t *testing.T) {
|
||||
cases := []struct {
|
||||
from string // page file
|
||||
to string // target file
|
||||
want string
|
||||
}{
|
||||
{"docs/intro/index.html", "docs/index.html", "../index.html"},
|
||||
{"docs/intro/index.html", "index.html", "../../index.html"},
|
||||
{"docs/intro/index.html", "_kage/ex.com/css/main.css", "../../_kage/ex.com/css/main.css"},
|
||||
{"index.html", "docs/index.html", "docs/index.html"},
|
||||
{"index.html", "_kage/ex.com/x.png", "_kage/ex.com/x.png"},
|
||||
{"a/b/c/index.html", "a/b/c/index.html", "index.html"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := Rel(Dir(c.from), c.to)
|
||||
if got != c.want {
|
||||
t.Errorf("Rel(dir(%q), %q) = %q, want %q", c.from, c.to, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRelResolves checks the invariant that follows: joining a page's directory
|
||||
// with the relative link Rel produced lands exactly on the target file.
|
||||
func TestRelResolves(t *testing.T) {
|
||||
pairs := [][2]string{
|
||||
{"docs/intro/index.html", "_kage/cdn.io/a/b.css"},
|
||||
{"index.html", "deep/a/b/c/index.html"},
|
||||
{"x/y/index.html", "x/index.html"},
|
||||
}
|
||||
for _, p := range pairs {
|
||||
from, to := p[0], p[1]
|
||||
rel := Rel(Dir(from), to)
|
||||
joined := joinAndClean(Dir(from), rel)
|
||||
if joined != to {
|
||||
t.Errorf("Rel round-trip: from=%q to=%q rel=%q joined=%q", from, to, rel, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func joinAndClean(dir, rel string) string {
|
||||
full := dir + "/" + rel
|
||||
parts := splitNonEmpty(full)
|
||||
var stack []string
|
||||
for _, p := range parts {
|
||||
switch p {
|
||||
case ".":
|
||||
case "..":
|
||||
if len(stack) > 0 {
|
||||
stack = stack[:len(stack)-1]
|
||||
}
|
||||
default:
|
||||
stack = append(stack, p)
|
||||
}
|
||||
}
|
||||
return strings.Join(stack, "/")
|
||||
}
|
||||
Reference in New Issue
Block a user