From e6afa91e09c42331d5bb52dfcf4d47b660706653 Mon Sep 17 00:00:00 2001 From: Duc-Tam Nguyen Date: Sun, 14 Jun 2026 18:22:25 +0700 Subject: [PATCH] Add the clone engine, CLI, tests, CI, and docs kage renders every page in headless Chrome, snapshots the final DOM, strips all JavaScript, and localises CSS, images, and fonts so a site can be browsed offline as a plain folder of files. The engine is split into small packages: urlx deterministic URL to local-path mapping and scope rules sanitize remove scripts, on* handlers, and javascript: URLs asset rewrite HTML and CSS references, download assets browser headless Chrome pool over the DevTools protocol robots robots.txt matcher clone the orchestrator: a polite resumable breadth-first crawl The cli package wires a cobra and fang command surface with two commands, clone and serve. Every pure package has table tests; the browser and clone packages add Chrome-driven end-to-end tests that skip when no browser is present or under -short. CI runs gofmt, vet, build, race tests, golangci-lint, govulncheck, and a tidy check on Linux and macOS. A goreleaser config fans one tag out to archives, deb/rpm/apk, a Chromium-bundled GHCR image, and the package managers. A tago docs site builds to Pages and Cloudflare. --- .github/workflows/ci.yml | 103 +++++ .github/workflows/docs.yml | 135 +++++++ .github/workflows/release.yml | 87 +++++ .goreleaser.yaml | 192 ++++++++++ Dockerfile | 36 ++ Makefile | 39 ++ asset/asset_test.go | 116 ++++++ asset/css.go | 69 ++++ asset/download.go | 83 ++++ asset/html.go | 147 ++++++++ browser/pool.go | 263 +++++++++++++ browser/pool_test.go | 52 +++ cli/clone.go | 190 ++++++++++ cli/root.go | 45 +++ cli/serve.go | 54 +++ cli/styles.go | 14 + cli/version.go | 9 + clone/cloner.go | 377 +++++++++++++++++++ clone/cloner_test.go | 226 +++++++++++ clone/config.go | 86 +++++ clone/frontier.go | 106 ++++++ clone/frontier_test.go | 67 ++++ clone/sitemap.go | 83 ++++ clone/stats.go | 37 ++ cmd/kage/main.go | 18 + docs/content/_index.md | 41 ++ docs/content/getting-started/_index.md | 11 + docs/content/getting-started/installation.md | 63 ++++ docs/content/getting-started/introduction.md | 53 +++ docs/content/getting-started/quick-start.md | 68 ++++ docs/content/guides/_index.md | 11 + docs/content/guides/resuming-a-run.md | 48 +++ docs/content/guides/scoping-a-crawl.md | 73 ++++ docs/content/guides/serving-a-mirror.md | 48 +++ docs/content/reference/_index.md | 11 + docs/content/reference/cli.md | 85 +++++ docs/content/reference/configuration.md | 54 +++ docs/static/favicon.svg | 1 + docs/tago.toml | 10 + go.mod | 44 +++ go.sum | 96 +++++ robots/robots.go | 218 +++++++++++ robots/robots_test.go | 73 ++++ sanitize/sanitize.go | 220 +++++++++++ sanitize/sanitize_test.go | 126 +++++++ scripts/ensure_cloudflare_pages.py | 144 +++++++ urlx/urlx.go | 318 ++++++++++++++++ urlx/urlx_test.go | 247 ++++++++++++ 48 files changed, 4697 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/release.yml create mode 100644 .goreleaser.yaml create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 asset/asset_test.go create mode 100644 asset/css.go create mode 100644 asset/download.go create mode 100644 asset/html.go create mode 100644 browser/pool.go create mode 100644 browser/pool_test.go create mode 100644 cli/clone.go create mode 100644 cli/root.go create mode 100644 cli/serve.go create mode 100644 cli/styles.go create mode 100644 cli/version.go create mode 100644 clone/cloner.go create mode 100644 clone/cloner_test.go create mode 100644 clone/config.go create mode 100644 clone/frontier.go create mode 100644 clone/frontier_test.go create mode 100644 clone/sitemap.go create mode 100644 clone/stats.go create mode 100644 cmd/kage/main.go create mode 100644 docs/content/_index.md create mode 100644 docs/content/getting-started/_index.md create mode 100644 docs/content/getting-started/installation.md create mode 100644 docs/content/getting-started/introduction.md create mode 100644 docs/content/getting-started/quick-start.md create mode 100644 docs/content/guides/_index.md create mode 100644 docs/content/guides/resuming-a-run.md create mode 100644 docs/content/guides/scoping-a-crawl.md create mode 100644 docs/content/guides/serving-a-mirror.md create mode 100644 docs/content/reference/_index.md create mode 100644 docs/content/reference/cli.md create mode 100644 docs/content/reference/configuration.md create mode 100644 docs/static/favicon.svg create mode 100644 docs/tago.toml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 robots/robots.go create mode 100644 robots/robots_test.go create mode 100644 sanitize/sanitize.go create mode 100644 sanitize/sanitize_test.go create mode 100755 scripts/ensure_cloudflare_pages.py create mode 100644 urlx/urlx.go create mode 100644 urlx/urlx_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e8fd589 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,103 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +# Cancel an in-flight run when a branch is pushed again. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Build and test on Linux and macOS with the race detector on. gofmt and vet + # run here too so a formatting slip fails fast on both platforms. The full + # suite includes Chrome-driven end-to-end tests, so the runners install a + # browser first; tests that find none skip themselves, so the job still passes + # if a future runner image drops Chrome. + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - name: install chromium (linux) + if: matrix.os == 'ubuntu-latest' + uses: browser-actions/setup-chrome@v1 + id: chrome + - name: gofmt + run: | + unformatted=$(gofmt -s -l asset browser cli clone cmd robots sanitize urlx) + if [ -n "$unformatted" ]; then + echo "These files need gofmt -s -w:" + echo "$unformatted" + exit 1 + fi + - name: go vet + run: go vet ./... + - name: build + run: go build ./... + - name: test + env: + KAGE_CHROME: ${{ steps.chrome.outputs.chrome-path }} + run: go test -race -count=1 -coverprofile=coverage.out ./... + - name: coverage summary + if: matrix.os == 'ubuntu-latest' + run: go tool cover -func=coverage.out | tail -1 + + # golangci-lint bundles staticcheck, govet, ineffassign, errcheck, unused and + # more, so it is the main quality gate. + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - uses: golangci/golangci-lint-action@v8 + with: + version: latest + + # Scan the module and its dependencies for known vulnerabilities. + govulncheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - name: govulncheck + run: | + go install golang.org/x/vuln/cmd/govulncheck@latest + govulncheck ./... + + # Confirm go.mod and go.sum are tidy: a PR that adds an import without running + # go mod tidy fails here instead of breaking a later release. + tidy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - name: go mod tidy is clean + run: | + go mod tidy + git diff --exit-code -- go.mod go.sum diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..7296c29 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,135 @@ +name: Docs + +on: + push: + branches: [main] + paths: + - "docs/**" + - ".github/workflows/docs.yml" + - "scripts/ensure_cloudflare_pages.py" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + deployments: write + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +concurrency: + group: docs-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6.0.2 + with: + submodules: true + # Sitemap lastmod comes from the latest content commit. + fetch-depth: 0 + + - name: Checkout tago + uses: actions/checkout@v6.0.2 + with: + repository: tamnd/tago + path: .tago-src + fetch-depth: 1 + + - name: Setup Go + uses: actions/setup-go@v6.4.0 + with: + go-version-file: .tago-src/go.mod + cache: false + + - name: Cache tago binary + id: tago-bin-cache + uses: actions/cache@v5.0.5 + with: + path: /usr/local/bin/tago + key: tago-bin-${{ runner.os }}-${{ hashFiles('.tago-src/go.sum', '.tago-src/**/*.go') }} + + - name: Build tago + if: steps.tago-bin-cache.outputs.cache-hit != 'true' + run: | + cd .tago-src + go build -o /usr/local/bin/tago ./cmd/tago/ + + # Two builds, one per deploy target. The theme links every page and asset + # with absURL, so each output carries its own base path: the Pages build + # gets the /kage/ sub-path, the Cloudflare build gets the domain root. + - name: Build for GitHub Pages + working-directory: docs + run: tago build --base-url "https://tamnd.github.io/kage/" --output public-pages + + # tago absURLs theme links but passes markdown content links through, so + # root-relative links in content need the sub-path prefixed for the Pages + # mirror. The Cloudflare build serves from the root and needs none. + - name: Rewrite content links for the Pages sub-path + run: | + find docs/public-pages -name '*.html' -print0 | + xargs -0 perl -pi -e 's|(href=")/(?!/)|${1}/kage/|g; s|(src=")/(?!/)|${1}/kage/|g' + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v5.0.0 + with: + path: docs/public-pages + + - name: Build for Cloudflare Pages + working-directory: docs + run: tago build --base-url "https://kage.tamnd.com/" --output public-cf + + - name: Upload Cloudflare artifact + uses: actions/upload-artifact@v7.0.1 + with: + name: public-cf + path: docs/public-cf + retention-days: 1 + + deploy-pages: + runs-on: ubuntu-latest + needs: build + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v5.0.0 + + deploy-cloudflare: + runs-on: ubuntu-latest + needs: build + concurrency: + group: cloudflare-pages-kage + cancel-in-progress: true + steps: + - uses: actions/checkout@v6.0.2 + with: + fetch-depth: 1 + sparse-checkout: scripts/ + + - name: Download Cloudflare artifact + uses: actions/download-artifact@v8.0.1 + with: + name: public-cf + path: public-cf + + - name: Ensure Cloudflare Pages config + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: python3 scripts/ensure_cloudflare_pages.py --project kage --domain kage.tamnd.com --zone tamnd.com + + - name: Deploy to Cloudflare Pages + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + npx -y wrangler@4 pages deploy public-cf \ + --project-name=kage \ + --branch=main \ + --commit-dirty=true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..fa9747d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,87 @@ +name: release + +# Pushing a version tag turns the GoReleaser config (.goreleaser.yaml) into a +# GitHub release with the archives, Linux packages (deb, rpm, apk), checksums, +# SBOMs and a cosign signature, and pushes the multi-arch container image to +# GHCR. Pull requests and pushes to main run `goreleaser check` only, so a +# config that would fail a real release is caught long before the tag. + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Fast gate on every PR and push: the config parses and is valid for the + # installed GoReleaser version. No artifacts are built here. + check: + if: github.ref_type != 'tag' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: "~> v2" + args: check + + # The real release, only on a version tag. + release: + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + contents: write # create the GitHub release + packages: write # push the image to ghcr.io + id-token: write # keyless cosign signing + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + + # Build and ship the linux/arm64 image from the amd64 runner. + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Tools GoReleaser shells out to for signing and SBOMs. + - uses: sigstore/cosign-installer@v3 + - uses: anchore/sbom-action/download-syft@v0 + + - uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: "~> v2" + args: release --clean + env: + # Creating the release and pushing the GHCR image use the built-in + # token. The package-manager publish steps each read their own secret; + # any that is unset leaves that manager skipped (the artifact is still + # produced), so the release never fails for a tap that is not set up. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} + SCOOP_BUCKET_GITHUB_TOKEN: ${{ secrets.SCOOP_BUCKET_GITHUB_TOKEN }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..b384a4e --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,192 @@ +# GoReleaser turns one tag push into everything a user might install from: raw +# archives, Linux packages (deb, rpm, apk), a multi-arch container image, and +# entries for the package managers (Homebrew, Scoop). `git tag vX.Y.Z && git +# push --tags` fans out to all of them through .github/workflows/release.yml. +# +# Publish steps that push to a repository we do not own yet (the Homebrew tap, +# the Scoop bucket) self-disable when their token is absent. A release with no +# extra secrets still produces every downloadable artifact and the container +# image; each manager lights up the moment its repository and token exist. +version: 2 + +project_name: kage + +before: + # Only fetch modules; never `go mod tidy` during a release, the tree is kept + # tidy in CI. + hooks: + - go mod download + +builds: + - id: kage + binary: kage + main: ./cmd/kage + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w + - -X github.com/tamnd/kage/cli.Version={{ .Version }} + - -X github.com/tamnd/kage/cli.Commit={{ .ShortCommit }} + - -X github.com/tamnd/kage/cli.Date={{ .CommitDate }} + mod_timestamp: "{{ .CommitTimestamp }}" + targets: + - linux_amd64 + - linux_arm64 + - linux_arm_7 + - linux_386 + - darwin_amd64 + - darwin_arm64 + - windows_amd64 + - windows_arm64 + - freebsd_amd64 + - freebsd_arm64 + +archives: + # tar.gz everywhere except a zip on Windows. + - id: default + name_template: "kage_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}" + format_overrides: + - goos: windows + formats: [zip] + files: + - LICENSE + - README.md + +nfpms: + # One nfpm definition emits the deb, rpm, and apk for every Linux build. kage + # is a user command, not a daemon, so there is no unit file and no + # postinstall: the package is the binary and its license. Chrome is a runtime + # dependency the user supplies (or the container image bundles). + - id: linux-packages + package_name: kage + file_name_template: "{{ .ConventionalFileName }}" + vendor: tamnd + homepage: https://github.com/tamnd/kage + maintainer: Duc-Tam Nguyen + description: Clone any website for offline viewing, with the JavaScript stripped out. + license: MIT + formats: + - deb + - rpm + - apk + bindir: /usr/bin + section: utils + recommends: + - chromium + contents: + - src: ./LICENSE + dst: /usr/share/doc/kage/LICENSE + +dockers_v2: + # One multi-platform image built with buildx. GoReleaser stages the prebuilt + # binaries under per-platform directories in the build context and the + # Dockerfile copies the right one through $TARGETPLATFORM. + - images: + - ghcr.io/tamnd/kage + tags: + - "{{ .Version }}" + - latest + dockerfile: Dockerfile + platforms: + - linux/amd64 + - linux/arm64 + labels: + org.opencontainers.image.title: "{{ .ProjectName }}" + org.opencontainers.image.description: "Clone any website for offline viewing, with the JavaScript stripped out" + org.opencontainers.image.url: "https://github.com/tamnd/kage" + org.opencontainers.image.source: "https://github.com/tamnd/kage" + org.opencontainers.image.version: "{{ .Version }}" + org.opencontainers.image.revision: "{{ .FullCommit }}" + org.opencontainers.image.licenses: "MIT" + +homebrew_casks: + # Homebrew cask pushed to the tap repository. Self-disables until + # HOMEBREW_TAP_GITHUB_TOKEN (a PAT with write to tamnd/homebrew-tap) is set, + # so a tokenless release still writes the cask into dist for inspection. + - name: kage + repository: + owner: tamnd + name: homebrew-tap + token: '{{ envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}' + directory: Casks + homepage: https://github.com/tamnd/kage + description: Clone any website for offline viewing, with the JavaScript stripped out + skip_upload: '{{ if envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}' + commit_author: + name: Duc-Tam Nguyen + email: tamnd87@gmail.com + +scoops: + # Scoop manifest for Windows, pushed to the bucket repository. + - repository: + owner: tamnd + name: scoop-bucket + token: '{{ envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}' + homepage: https://github.com/tamnd/kage + description: Clone any website for offline viewing, with the JavaScript stripped out + license: MIT + skip_upload: '{{ if envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}' + commit_author: + name: Duc-Tam Nguyen + email: tamnd87@gmail.com + +checksum: + name_template: "checksums.txt" + algorithm: sha256 + +sboms: + # A CycloneDX SBOM per archive via syft; the release workflow installs it. + - id: archive + artifacts: archive + +signs: + # Keyless cosign signature over the checksum file. It runs only on a real + # release in CI, where the workflow grants the OIDC token cosign needs. + - cmd: cosign + certificate: "${artifact}.pem" + args: + - sign-blob + - "--output-certificate=${certificate}" + - "--output-signature=${signature}" + - "${artifact}" + - "--yes" + artifacts: checksum + output: true + +docker_signs: + - cmd: cosign + artifacts: manifests + args: + - sign + - "${artifact}@${digest}" + - "--yes" + +changelog: + sort: asc + use: github + filters: + exclude: + - "^docs:" + - "^test:" + - "^chore:" + - "^ci:" + - Merge pull request + - Merge branch + groups: + - title: Features + regexp: '^.*?feat(\(.+\))??!?:.+$' + order: 0 + - title: Fixes + regexp: '^.*?fix(\(.+\))??!?:.+$' + order: 1 + - title: Other + order: 999 + +release: + github: + owner: tamnd + name: kage + draft: false + prerelease: auto diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7b20f98 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# Consumed by GoReleaser: it copies the already cross-compiled binary out of the +# build context rather than compiling, so the image build is fast and uses the +# same static binary every other artifact ships. +# +# kage always drives a real headless Chrome, so unlike a plain CLI image this one +# bundles Chromium. KAGE_CHROME points kage at the system binary so it never +# tries to download its own. +# +# GoReleaser builds one multi-platform image with buildx and stages each +# platform's binary under a $TARGETPLATFORM directory (e.g. linux/amd64/) in the +# build context, so the COPY line selects the right one through the automatic +# TARGETPLATFORM build arg. +FROM alpine:3.21 + +ARG TARGETPLATFORM + +# chromium for rendering; ca-certificates for HTTPS; tzdata for sane timestamps; +# the font package so rendered pages have glyphs to lay out. +RUN apk add --no-cache chromium ca-certificates tzdata font-noto \ + && adduser -D -H -u 10001 kage \ + && mkdir -p /out \ + && chown kage:kage /out + +COPY $TARGETPLATFORM/kage /usr/bin/kage + +USER kage +WORKDIR /out + +# Point kage at the bundled Chromium and write mirrors under /out by default: +# +# docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com +ENV KAGE_CHROME=/usr/bin/chromium-browser + +VOLUME ["/out"] + +ENTRYPOINT ["/usr/bin/kage"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..099483f --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +BIN := kage +PKG := ./cmd/kage +VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo none) +DATE := $(shell date -u +%Y-%m-%dT%H:%M:%SZ) +LDFLAGS := -s -w \ + -X github.com/tamnd/kage/cli.Version=$(VERSION) \ + -X github.com/tamnd/kage/cli.Commit=$(COMMIT) \ + -X github.com/tamnd/kage/cli.Date=$(DATE) + +export CGO_ENABLED := 0 + +.PHONY: build install test test-short vet tidy clean run + +build: + go build -ldflags "$(LDFLAGS)" -o bin/$(BIN) $(PKG) + +install: + go install -ldflags "$(LDFLAGS)" $(PKG) + +# Full suite, including the Chrome-driven end-to-end tests. +test: + go test -race ./... + +# Skip the tests that launch a real browser (CI without Chrome, quick loops). +test-short: + go test -short ./... + +vet: + go vet ./... + +tidy: + go mod tidy + +clean: + rm -rf bin + +run: build + ./bin/$(BIN) diff --git a/asset/asset_test.go b/asset/asset_test.go new file mode 100644 index 0000000..672cc64 --- /dev/null +++ b/asset/asset_test.go @@ -0,0 +1,116 @@ +package asset + +import ( + "bytes" + "net/url" + "sort" + "strings" + "testing" + + "github.com/tamnd/kage/urlx" + "golang.org/x/net/html" +) + +// testSink builds a RefSink that records every URL it sees and rewrites it to a +// local relative path from fromDir, mirroring what the cloner does. +func testSink(seedHost, fromDir string, seen *[]string) RefSink { + return func(u *url.URL, kind urlx.Kind) string { + *seen = append(*seen, u.String()) + if !urlx.SameSite(&url.URL{Host: seedHost}, u, true) && kind == urlx.Page { + return u.String() // external page links stay absolute + } + local := urlx.LocalPath(seedHost, u, kind, "") + return urlx.Rel(fromDir, local) + } +} + +func TestRewriteCSS(t *testing.T) { + base, _ := url.Parse("https://ex.com/css/main.css") + in := `@import "base.css"; +@import url('https://cdn.io/reset.css'); +body { background: url(../img/bg.png); } +.x { background: url("https://cdn.io/sprite.svg"); } +.y { content: url(data:image/gif;base64,AAAA); }` + var seen []string + // CSS lives at _kage/ex.com/css/main.css → its dir. + fromDir := "_kage/ex.com/css" + out := string(RewriteCSS([]byte(in), base, testSink("ex.com", fromDir, &seen))) + + if !strings.Contains(out, `@import "base.css"`) { + t.Errorf("local @import not rewritten relative: %s", out) + } + if !strings.Contains(out, `url("../img/bg.png")`) { + t.Errorf("local url() not rewritten relative: %s", out) + } + // Cross-origin asset goes under _kage/cdn.io/...; from _kage/ex.com/css that + // is ../../cdn.io/... + if !strings.Contains(out, `url("../../cdn.io/reset.css")`) { + t.Errorf("cross-origin @import not localised: %s", out) + } + if !strings.Contains(out, `url("../../cdn.io/sprite.svg")`) { + t.Errorf("cross-origin url() not localised: %s", out) + } + // data: URL must be left exactly as-is. + if !strings.Contains(out, "url(data:image/gif;base64,AAAA)") { + t.Errorf("data: URL was altered: %s", out) + } + + sort.Strings(seen) + want := []string{ + "https://cdn.io/reset.css", + "https://cdn.io/sprite.svg", + "https://ex.com/css/base.css", + "https://ex.com/img/bg.png", + } + if strings.Join(seen, ",") != strings.Join(want, ",") { + t.Errorf("sink saw %v, want %v", seen, want) + } +} + +const htmlDoc = ` + + + + +internal +external +a pdf + +

hi

+ +` + +func TestRewriteHTML(t *testing.T) { + base, _ := url.Parse("https://ex.com/") + root, err := html.Parse(strings.NewReader(htmlDoc)) + if err != nil { + t.Fatal(err) + } + var seen []string + // The page is the root index.html, so fromDir is "". + RewriteHTML(root, base, testSink("ex.com", "", &seen)) + + var buf bytes.Buffer + if err := html.Render(&buf, root); err != nil { + t.Fatal(err) + } + out := buf.String() + + checks := map[string]bool{ + `href="_kage/ex.com/css/main.css"`: true, // stylesheet localised + `href="_kage/ex.com/favicon.ico"`: true, // icon localised + `href="https://ex.com/canon"`: true, // canonical left alone + `href="docs/intro/index.html"`: true, // internal page → local + `href="https://other.com/x"`: true, // external page stays absolute + `href="_kage/ex.com/files/report.pdf"`: true, // pdf link treated as asset + `src="_kage/ex.com/img/logo.png"`: true, + `_kage/ex.com/img/logo@2x.png 2x`: true, // srcset candidate rewritten + `background:url("_kage/ex.com/img/bg.png")`: true, // inline style (attr-escaped quotes) + `background:url("_kage/ex.com/img/tile.png")`: true, // + + + + +

Title

+js link +real link + +
+ +

styled

+` + +func TestStripRemovesAllJS(t *testing.T) { + out, rep, err := Strip([]byte(page), Options{Banner: "cloned by kage"}) + if err != nil { + t.Fatal(err) + } + s := string(out) + + if strings.Contains(s, " survived") + } + if strings.Contains(s, "onload") || strings.Contains(s, "onclick") || strings.Contains(s, "onerror") { + t.Error("an on* handler survived") + } + if strings.Contains(strings.ToLower(s), "javascript:") { + t.Error("a javascript: URL survived") + } + if strings.Contains(s, "modulepreload") || strings.Contains(s, "preconnect") { + t.Error("a dead resource hint survived") + } + if strings.Contains(s, "http-equiv") { + t.Error("a meta refresh survived") + } + + // Layout-bearing markup must survive untouched. + for _, want := range []string{ + `rel="stylesheet"`, `href="/css/main.css"`, + `