diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e8fd589 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,103 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +# Cancel an in-flight run when a branch is pushed again. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Build and test on Linux and macOS with the race detector on. gofmt and vet + # run here too so a formatting slip fails fast on both platforms. The full + # suite includes Chrome-driven end-to-end tests, so the runners install a + # browser first; tests that find none skip themselves, so the job still passes + # if a future runner image drops Chrome. + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - name: install chromium (linux) + if: matrix.os == 'ubuntu-latest' + uses: browser-actions/setup-chrome@v1 + id: chrome + - name: gofmt + run: | + unformatted=$(gofmt -s -l asset browser cli clone cmd robots sanitize urlx) + if [ -n "$unformatted" ]; then + echo "These files need gofmt -s -w:" + echo "$unformatted" + exit 1 + fi + - name: go vet + run: go vet ./... + - name: build + run: go build ./... + - name: test + env: + KAGE_CHROME: ${{ steps.chrome.outputs.chrome-path }} + run: go test -race -count=1 -coverprofile=coverage.out ./... + - name: coverage summary + if: matrix.os == 'ubuntu-latest' + run: go tool cover -func=coverage.out | tail -1 + + # golangci-lint bundles staticcheck, govet, ineffassign, errcheck, unused and + # more, so it is the main quality gate. + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - uses: golangci/golangci-lint-action@v8 + with: + version: latest + + # Scan the module and its dependencies for known vulnerabilities. + govulncheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - name: govulncheck + run: | + go install golang.org/x/vuln/cmd/govulncheck@latest + govulncheck ./... + + # Confirm go.mod and go.sum are tidy: a PR that adds an import without running + # go mod tidy fails here instead of breaking a later release. + tidy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - name: go mod tidy is clean + run: | + go mod tidy + git diff --exit-code -- go.mod go.sum diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..7296c29 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,135 @@ +name: Docs + +on: + push: + branches: [main] + paths: + - "docs/**" + - ".github/workflows/docs.yml" + - "scripts/ensure_cloudflare_pages.py" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + deployments: write + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +concurrency: + group: docs-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6.0.2 + with: + submodules: true + # Sitemap lastmod comes from the latest content commit. + fetch-depth: 0 + + - name: Checkout tago + uses: actions/checkout@v6.0.2 + with: + repository: tamnd/tago + path: .tago-src + fetch-depth: 1 + + - name: Setup Go + uses: actions/setup-go@v6.4.0 + with: + go-version-file: .tago-src/go.mod + cache: false + + - name: Cache tago binary + id: tago-bin-cache + uses: actions/cache@v5.0.5 + with: + path: /usr/local/bin/tago + key: tago-bin-${{ runner.os }}-${{ hashFiles('.tago-src/go.sum', '.tago-src/**/*.go') }} + + - name: Build tago + if: steps.tago-bin-cache.outputs.cache-hit != 'true' + run: | + cd .tago-src + go build -o /usr/local/bin/tago ./cmd/tago/ + + # Two builds, one per deploy target. The theme links every page and asset + # with absURL, so each output carries its own base path: the Pages build + # gets the /kage/ sub-path, the Cloudflare build gets the domain root. + - name: Build for GitHub Pages + working-directory: docs + run: tago build --base-url "https://tamnd.github.io/kage/" --output public-pages + + # tago absURLs theme links but passes markdown content links through, so + # root-relative links in content need the sub-path prefixed for the Pages + # mirror. The Cloudflare build serves from the root and needs none. + - name: Rewrite content links for the Pages sub-path + run: | + find docs/public-pages -name '*.html' -print0 | + xargs -0 perl -pi -e 's|(href=")/(?!/)|${1}/kage/|g; s|(src=")/(?!/)|${1}/kage/|g' + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v5.0.0 + with: + path: docs/public-pages + + - name: Build for Cloudflare Pages + working-directory: docs + run: tago build --base-url "https://kage.tamnd.com/" --output public-cf + + - name: Upload Cloudflare artifact + uses: actions/upload-artifact@v7.0.1 + with: + name: public-cf + path: docs/public-cf + retention-days: 1 + + deploy-pages: + runs-on: ubuntu-latest + needs: build + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v5.0.0 + + deploy-cloudflare: + runs-on: ubuntu-latest + needs: build + concurrency: + group: cloudflare-pages-kage + cancel-in-progress: true + steps: + - uses: actions/checkout@v6.0.2 + with: + fetch-depth: 1 + sparse-checkout: scripts/ + + - name: Download Cloudflare artifact + uses: actions/download-artifact@v8.0.1 + with: + name: public-cf + path: public-cf + + - name: Ensure Cloudflare Pages config + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: python3 scripts/ensure_cloudflare_pages.py --project kage --domain kage.tamnd.com --zone tamnd.com + + - name: Deploy to Cloudflare Pages + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + npx -y wrangler@4 pages deploy public-cf \ + --project-name=kage \ + --branch=main \ + --commit-dirty=true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..fa9747d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,87 @@ +name: release + +# Pushing a version tag turns the GoReleaser config (.goreleaser.yaml) into a +# GitHub release with the archives, Linux packages (deb, rpm, apk), checksums, +# SBOMs and a cosign signature, and pushes the multi-arch container image to +# GHCR. Pull requests and pushes to main run `goreleaser check` only, so a +# config that would fail a real release is caught long before the tag. + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Fast gate on every PR and push: the config parses and is valid for the + # installed GoReleaser version. No artifacts are built here. + check: + if: github.ref_type != 'tag' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + - uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: "~> v2" + args: check + + # The real release, only on a version tag. + release: + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + contents: write # create the GitHub release + packages: write # push the image to ghcr.io + id-token: write # keyless cosign signing + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + + # Build and ship the linux/arm64 image from the amd64 runner. + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Tools GoReleaser shells out to for signing and SBOMs. + - uses: sigstore/cosign-installer@v3 + - uses: anchore/sbom-action/download-syft@v0 + + - uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: "~> v2" + args: release --clean + env: + # Creating the release and pushing the GHCR image use the built-in + # token. The package-manager publish steps each read their own secret; + # any that is unset leaves that manager skipped (the artifact is still + # produced), so the release never fails for a tap that is not set up. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} + SCOOP_BUCKET_GITHUB_TOKEN: ${{ secrets.SCOOP_BUCKET_GITHUB_TOKEN }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..b384a4e --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,192 @@ +# GoReleaser turns one tag push into everything a user might install from: raw +# archives, Linux packages (deb, rpm, apk), a multi-arch container image, and +# entries for the package managers (Homebrew, Scoop). `git tag vX.Y.Z && git +# push --tags` fans out to all of them through .github/workflows/release.yml. +# +# Publish steps that push to a repository we do not own yet (the Homebrew tap, +# the Scoop bucket) self-disable when their token is absent. A release with no +# extra secrets still produces every downloadable artifact and the container +# image; each manager lights up the moment its repository and token exist. +version: 2 + +project_name: kage + +before: + # Only fetch modules; never `go mod tidy` during a release, the tree is kept + # tidy in CI. + hooks: + - go mod download + +builds: + - id: kage + binary: kage + main: ./cmd/kage + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w + - -X github.com/tamnd/kage/cli.Version={{ .Version }} + - -X github.com/tamnd/kage/cli.Commit={{ .ShortCommit }} + - -X github.com/tamnd/kage/cli.Date={{ .CommitDate }} + mod_timestamp: "{{ .CommitTimestamp }}" + targets: + - linux_amd64 + - linux_arm64 + - linux_arm_7 + - linux_386 + - darwin_amd64 + - darwin_arm64 + - windows_amd64 + - windows_arm64 + - freebsd_amd64 + - freebsd_arm64 + +archives: + # tar.gz everywhere except a zip on Windows. + - id: default + name_template: "kage_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}" + format_overrides: + - goos: windows + formats: [zip] + files: + - LICENSE + - README.md + +nfpms: + # One nfpm definition emits the deb, rpm, and apk for every Linux build. kage + # is a user command, not a daemon, so there is no unit file and no + # postinstall: the package is the binary and its license. Chrome is a runtime + # dependency the user supplies (or the container image bundles). + - id: linux-packages + package_name: kage + file_name_template: "{{ .ConventionalFileName }}" + vendor: tamnd + homepage: https://github.com/tamnd/kage + maintainer: Duc-Tam Nguyen + description: Clone any website for offline viewing, with the JavaScript stripped out. + license: MIT + formats: + - deb + - rpm + - apk + bindir: /usr/bin + section: utils + recommends: + - chromium + contents: + - src: ./LICENSE + dst: /usr/share/doc/kage/LICENSE + +dockers_v2: + # One multi-platform image built with buildx. GoReleaser stages the prebuilt + # binaries under per-platform directories in the build context and the + # Dockerfile copies the right one through $TARGETPLATFORM. + - images: + - ghcr.io/tamnd/kage + tags: + - "{{ .Version }}" + - latest + dockerfile: Dockerfile + platforms: + - linux/amd64 + - linux/arm64 + labels: + org.opencontainers.image.title: "{{ .ProjectName }}" + org.opencontainers.image.description: "Clone any website for offline viewing, with the JavaScript stripped out" + org.opencontainers.image.url: "https://github.com/tamnd/kage" + org.opencontainers.image.source: "https://github.com/tamnd/kage" + org.opencontainers.image.version: "{{ .Version }}" + org.opencontainers.image.revision: "{{ .FullCommit }}" + org.opencontainers.image.licenses: "MIT" + +homebrew_casks: + # Homebrew cask pushed to the tap repository. Self-disables until + # HOMEBREW_TAP_GITHUB_TOKEN (a PAT with write to tamnd/homebrew-tap) is set, + # so a tokenless release still writes the cask into dist for inspection. + - name: kage + repository: + owner: tamnd + name: homebrew-tap + token: '{{ envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}' + directory: Casks + homepage: https://github.com/tamnd/kage + description: Clone any website for offline viewing, with the JavaScript stripped out + skip_upload: '{{ if envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}' + commit_author: + name: Duc-Tam Nguyen + email: tamnd87@gmail.com + +scoops: + # Scoop manifest for Windows, pushed to the bucket repository. + - repository: + owner: tamnd + name: scoop-bucket + token: '{{ envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}' + homepage: https://github.com/tamnd/kage + description: Clone any website for offline viewing, with the JavaScript stripped out + license: MIT + skip_upload: '{{ if envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}' + commit_author: + name: Duc-Tam Nguyen + email: tamnd87@gmail.com + +checksum: + name_template: "checksums.txt" + algorithm: sha256 + +sboms: + # A CycloneDX SBOM per archive via syft; the release workflow installs it. + - id: archive + artifacts: archive + +signs: + # Keyless cosign signature over the checksum file. It runs only on a real + # release in CI, where the workflow grants the OIDC token cosign needs. + - cmd: cosign + certificate: "${artifact}.pem" + args: + - sign-blob + - "--output-certificate=${certificate}" + - "--output-signature=${signature}" + - "${artifact}" + - "--yes" + artifacts: checksum + output: true + +docker_signs: + - cmd: cosign + artifacts: manifests + args: + - sign + - "${artifact}@${digest}" + - "--yes" + +changelog: + sort: asc + use: github + filters: + exclude: + - "^docs:" + - "^test:" + - "^chore:" + - "^ci:" + - Merge pull request + - Merge branch + groups: + - title: Features + regexp: '^.*?feat(\(.+\))??!?:.+$' + order: 0 + - title: Fixes + regexp: '^.*?fix(\(.+\))??!?:.+$' + order: 1 + - title: Other + order: 999 + +release: + github: + owner: tamnd + name: kage + draft: false + prerelease: auto diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b8c941f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,43 @@ +# Changelog + +All notable changes to kage are recorded here. The format follows +[Keep a Changelog](https://keepachangelog.com/), and the project aims to follow +[Semantic Versioning](https://semver.org/). + +## [Unreleased] + +## [0.1.0] - 2026-06-14 + +The first release. kage clones a live website into a self-contained folder you +can browse offline, with every script stripped out. + +### Added + +- `kage clone ` renders each page in headless Chrome, snapshots the final + DOM, removes every ` +`)) + })) + defer srv.Close() + + p := New(Options{Headless: true, Workers: 1, Settle: 300 * time.Millisecond, RenderTimeout: 20 * time.Second}) + defer func() { _ = p.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + res, err := p.Render(ctx, srv.URL) + if err != nil { + t.Fatalf("render: %v", err) + } + if !strings.Contains(res.HTML, "rendered-by-js") { + t.Errorf("render did not capture the JS-built DOM:\n%s", res.HTML) + } +} diff --git a/cli/clone.go b/cli/clone.go new file mode 100644 index 0000000..7d221dc --- /dev/null +++ b/cli/clone.go @@ -0,0 +1,194 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + + "github.com/tamnd/kage/clone" + "github.com/tamnd/kage/urlx" +) + +// cloneFlags holds the parsed flag values for one invocation. +type cloneFlags struct { + out string + reserved string + workers int + assetWorkers int + browserPages int + maxPages int + maxDepth int + traversal string + maxAssetMB int64 + timeout time.Duration + settle time.Duration + renderTO time.Duration + scroll bool + userAgent string + subdomains bool + scopePrefix string + exclude []string + noRobots bool + noSitemap bool + headful bool + keepNoscript bool + chromeBin string + controlURL string + noResume bool + refresh bool + force bool + quiet bool +} + +func newCloneCmd() *cobra.Command { + f := &cloneFlags{} + cmd := &cobra.Command{ + Use: "clone ", + Short: "Clone a site into a self-contained offline folder", + Long: "clone fetches the seed URL, follows in-scope links, and writes a browsable\n" + + "mirror to //. Every page is rendered in Chrome and stripped of\n" + + "scripts; CSS, images, and fonts are downloaded and rewritten to local paths.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runClone(cmd.Context(), args[0], f) + }, + } + fs := cmd.Flags() + fs.StringVarP(&f.out, "out", "o", clone.DefaultOutDir(), "output root; the mirror lands in //") + fs.StringVar(&f.reserved, "reserved", urlx.DefaultReserved, "reserved dir for assets and state") + fs.IntVar(&f.workers, "workers", 4, "concurrent page render workers") + fs.IntVar(&f.assetWorkers, "asset-workers", 8, "concurrent asset download workers") + fs.IntVar(&f.browserPages, "browser-pages", 4, "Chrome page-pool size") + fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "stop after N pages (0 = unlimited)") + fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)") + fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order: bfs or dfs") + fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB") + fs.DurationVar(&f.timeout, "timeout", 30*time.Second, "per-request timeout") + fs.DurationVar(&f.settle, "settle", 1500*time.Millisecond, "network-idle quiet period before snapshot") + fs.DurationVar(&f.renderTO, "render-timeout", 30*time.Second, "hard cap per page render") + fs.BoolVar(&f.scroll, "scroll", false, "auto-scroll each page to trigger lazy loading") + fs.StringVar(&f.userAgent, "user-agent", clone.DefaultUserAgent, "User-Agent for asset and robots fetches") + fs.BoolVar(&f.subdomains, "subdomains", false, "treat subdomains of the seed host as in scope") + fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix") + fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)") + fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)") + fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml") + fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)") + fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap