Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8bc9c8bea1 | |||
| 28d08ade88 | |||
| 1e59c10e6f | |||
| 105e68f036 | |||
| 792878d6f3 | |||
| 910619bc7a | |||
| e0e4096f93 | |||
| c176719031 | |||
| a1b82ebd1b | |||
| 5ad8fff9df | |||
| 7157144c75 | |||
| de1c0adde3 | |||
| 16bbd1b9d0 | |||
| 3220f6d230 | |||
| 9d452ddf11 |
@@ -26,7 +26,7 @@ jobs:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
cache: true
|
||||
- name: install chromium (linux)
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
uses: browser-actions/setup-chrome@v1
|
||||
uses: browser-actions/setup-chrome@v2.1.2
|
||||
id: chrome
|
||||
- name: gofmt
|
||||
run: |
|
||||
@@ -67,13 +67,13 @@ jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: true
|
||||
- uses: golangci/golangci-lint-action@v8
|
||||
- uses: golangci/golangci-lint-action@v9.2.1
|
||||
with:
|
||||
version: latest
|
||||
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
govulncheck:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
tidy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
@@ -117,7 +117,7 @@ jobs:
|
||||
webview:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
@@ -133,7 +133,7 @@ jobs:
|
||||
windows-gui:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -26,14 +26,14 @@ jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6.0.2
|
||||
- uses: actions/checkout@v7.0.0
|
||||
with:
|
||||
submodules: true
|
||||
# Sitemap lastmod comes from the latest content commit.
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Checkout tago
|
||||
uses: actions/checkout@v6.0.2
|
||||
uses: actions/checkout@v7.0.0
|
||||
with:
|
||||
repository: tamnd/tago
|
||||
path: .tago-src
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
group: cloudflare-pages-kage
|
||||
cancel-in-progress: true
|
||||
steps:
|
||||
- uses: actions/checkout@v6.0.2
|
||||
- uses: actions/checkout@v7.0.0
|
||||
with:
|
||||
fetch-depth: 1
|
||||
sparse-checkout: scripts/
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
if: github.ref_type != 'tag'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-go@v6
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
cache: true
|
||||
- uses: goreleaser/goreleaser-action@v6
|
||||
- uses: goreleaser/goreleaser-action@v7.2.2
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: "~> v2"
|
||||
@@ -49,8 +49,10 @@ jobs:
|
||||
contents: write # create the GitHub release
|
||||
packages: write # push the image to ghcr.io
|
||||
id-token: write # keyless cosign signing
|
||||
env:
|
||||
LINUX_REPO_DISPATCH_TOKEN: ${{ secrets.LINUX_REPO_DISPATCH_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-go@v6
|
||||
@@ -60,19 +62,25 @@ jobs:
|
||||
cache: true
|
||||
|
||||
# Build and ship the linux/arm64 image from the amd64 runner.
|
||||
- uses: docker/setup-qemu-action@v3
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
- uses: docker/setup-qemu-action@v4.1.0
|
||||
- uses: docker/setup-buildx-action@v4.1.0
|
||||
- uses: docker/login-action@v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Tools GoReleaser shells out to for signing and SBOMs.
|
||||
- uses: sigstore/cosign-installer@v3
|
||||
- uses: anchore/sbom-action/download-syft@v0
|
||||
# Pin cosign to the 2.x line. cosign 3.x makes the new bundle format the
|
||||
# default, which ignores the --output-signature/--output-certificate flags
|
||||
# the signs block uses and aborts. Pinning keeps the .sig/.pem outputs and
|
||||
# stops the release tool from floating to a breaking latest.
|
||||
- uses: sigstore/cosign-installer@v4.1.2
|
||||
with:
|
||||
cosign-release: "v2.6.3"
|
||||
- uses: anchore/sbom-action/download-syft@v0.24.0
|
||||
|
||||
- uses: goreleaser/goreleaser-action@v6
|
||||
- uses: goreleaser/goreleaser-action@v7.2.2
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: "~> v2"
|
||||
@@ -85,3 +93,20 @@ jobs:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }}
|
||||
SCOOP_BUCKET_GITHUB_TOKEN: ${{ secrets.SCOOP_BUCKET_GITHUB_TOKEN }}
|
||||
|
||||
# Rebuild the Linux apt/dnf repository with the packages just released.
|
||||
# Skipped when the dispatch token is unset, so the release never depends
|
||||
# on it being configured.
|
||||
- name: Refresh the Linux package repository
|
||||
if: env.LINUX_REPO_DISPATCH_TOKEN != ''
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "Authorization: Bearer $LINUX_REPO_DISPATCH_TOKEN" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
https://api.github.com/repos/tamnd/linux-repo/dispatches \
|
||||
-d '{"event_type":"package-released"}' || true)
|
||||
if [ "$code" = "204" ]; then
|
||||
echo "Linux repo refresh triggered"
|
||||
else
|
||||
echo "Linux repo refresh not accepted (HTTP $code); skipping"
|
||||
fi
|
||||
|
||||
+5
-4
@@ -151,11 +151,11 @@ homebrew_casks:
|
||||
repository:
|
||||
owner: tamnd
|
||||
name: homebrew-tap
|
||||
token: '{{ envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}'
|
||||
token: '{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}'
|
||||
directory: Casks
|
||||
homepage: https://github.com/tamnd/kage
|
||||
description: Clone any website for offline viewing, with the JavaScript stripped out
|
||||
skip_upload: '{{ if envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}'
|
||||
skip_upload: '{{ if index .Env "HOMEBREW_TAP_GITHUB_TOKEN" }}false{{ else }}true{{ end }}'
|
||||
commit_author:
|
||||
name: Duc-Tam Nguyen
|
||||
email: tamnd87@gmail.com
|
||||
@@ -168,11 +168,12 @@ scoops:
|
||||
repository:
|
||||
owner: tamnd
|
||||
name: scoop-bucket
|
||||
token: '{{ envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}'
|
||||
token: '{{ .Env.SCOOP_BUCKET_GITHUB_TOKEN }}'
|
||||
directory: bucket
|
||||
homepage: https://github.com/tamnd/kage
|
||||
description: Clone any website for offline viewing, with the JavaScript stripped out
|
||||
license: MIT
|
||||
skip_upload: '{{ if envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}'
|
||||
skip_upload: '{{ if index .Env "SCOOP_BUCKET_GITHUB_TOKEN" }}false{{ else }}true{{ end }}'
|
||||
commit_author:
|
||||
name: Duc-Tam Nguyen
|
||||
email: tamnd87@gmail.com
|
||||
|
||||
@@ -24,7 +24,27 @@ Full docs and guides live at **[kage.tamnd.com](https://kage.tamnd.com)**.
|
||||
go install github.com/tamnd/kage/cmd/kage@latest
|
||||
```
|
||||
|
||||
Prefer a prebuilt binary? Grab an archive, a `.deb`/`.rpm`/`.apk`, or a checksum from [releases](https://github.com/tamnd/kage/releases). Or skip installing Chrome yourself and use the container image, which bundles Chromium:
|
||||
Prefer a prebuilt binary? Grab an archive, a `.deb`/`.rpm`/`.apk`, or a checksum from [releases](https://github.com/tamnd/kage/releases). Or let a package manager handle it:
|
||||
|
||||
```bash
|
||||
# Homebrew (macOS)
|
||||
brew install --cask tamnd/tap/kage
|
||||
|
||||
# Scoop (Windows)
|
||||
scoop bucket add tamnd https://github.com/tamnd/scoop-bucket
|
||||
scoop install kage
|
||||
|
||||
# apt (Debian, Ubuntu)
|
||||
curl -fsSL https://tamnd.github.io/linux-repo/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" | sudo tee /etc/apt/sources.list.d/tamnd.list
|
||||
sudo apt update && sudo apt install kage
|
||||
|
||||
# dnf (Fedora, RHEL)
|
||||
sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo
|
||||
sudo dnf install kage
|
||||
```
|
||||
|
||||
Or skip installing Chrome yourself and use the container image, which bundles Chromium:
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD/out:/out" ghcr.io/tamnd/kage clone paulgraham.com
|
||||
@@ -102,6 +122,7 @@ The flags you'll actually reach for:
|
||||
| `--scroll` | `false` | Auto-scroll each page to trigger lazy loading |
|
||||
| `--workers` | `4` | How many pages to render at once |
|
||||
| `--no-robots` | `false` | Ignore `robots.txt` (be nice) |
|
||||
| `--crawl-delay` | `0s` | Override robots.txt `Crawl-delay` between page starts |
|
||||
| `-f, --force` | `false` | Delete any existing mirror for the host first |
|
||||
| `--chrome` | | Path to the Chrome/Chromium binary |
|
||||
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
package browser
|
||||
|
||||
import "runtime"
|
||||
|
||||
func launcherLeakless() bool {
|
||||
return runtime.GOOS != "windows"
|
||||
}
|
||||
+1
-1
@@ -165,7 +165,7 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
|
||||
|
||||
controlURL := p.opts.ControlURL
|
||||
if controlURL == "" {
|
||||
l := launcher.New().
|
||||
l := launcher.New().Leakless(launcherLeakless()).
|
||||
Headless(p.opts.Headless).
|
||||
Set("disable-blink-features", "AutomationControlled").
|
||||
Set("disable-gpu", "")
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -83,6 +84,14 @@ func TestDisableSandboxContainer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLauncherLeaklessDisabledOnWindows(t *testing.T) {
|
||||
got := launcherLeakless()
|
||||
want := runtime.GOOS != "windows"
|
||||
if got != want {
|
||||
t.Errorf("launcherLeakless() = %v on %s; want %v", got, runtime.GOOS, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCapturesFinalDOM(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("render test drives Chrome; skipped under -short")
|
||||
|
||||
+37
-31
@@ -16,37 +16,38 @@ import (
|
||||
|
||||
// cloneFlags holds the parsed flag values for one invocation.
|
||||
type cloneFlags struct {
|
||||
out string
|
||||
reserved string
|
||||
workers int
|
||||
assetWorkers int
|
||||
browserPages int
|
||||
maxPages int
|
||||
maxDepth int
|
||||
traversal string
|
||||
maxAssetMB int64
|
||||
keepMedia bool
|
||||
skipExt []string
|
||||
allAssetHosts bool
|
||||
timeout time.Duration
|
||||
settle time.Duration
|
||||
renderTO time.Duration
|
||||
scroll bool
|
||||
userAgent string
|
||||
subdomains bool
|
||||
scopePrefix string
|
||||
exclude []string
|
||||
noRobots bool
|
||||
noSitemap bool
|
||||
headful bool
|
||||
keepNoscript bool
|
||||
mobileReadable bool
|
||||
chromeBin string
|
||||
controlURL string
|
||||
noResume bool
|
||||
refresh bool
|
||||
force bool
|
||||
quiet bool
|
||||
out string
|
||||
reserved string
|
||||
workers int
|
||||
assetWorkers int
|
||||
browserPages int
|
||||
maxPages int
|
||||
maxDepth int
|
||||
traversal string
|
||||
maxAssetMB int64
|
||||
keepMedia bool
|
||||
skipExt []string
|
||||
allAssetHosts bool
|
||||
timeout time.Duration
|
||||
settle time.Duration
|
||||
renderTO time.Duration
|
||||
scroll bool
|
||||
userAgent string
|
||||
subdomains bool
|
||||
scopePrefix string
|
||||
exclude []string
|
||||
noRobots bool
|
||||
crawlDelay time.Duration
|
||||
noSitemap bool
|
||||
headful bool
|
||||
keepNoscript bool
|
||||
mobileReadable bool
|
||||
chromeBin string
|
||||
controlURL string
|
||||
noResume bool
|
||||
refresh bool
|
||||
force bool
|
||||
quiet bool
|
||||
}
|
||||
|
||||
func newCloneCmd() *cobra.Command {
|
||||
@@ -84,6 +85,7 @@ func newCloneCmd() *cobra.Command {
|
||||
fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix")
|
||||
fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)")
|
||||
fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)")
|
||||
fs.DurationVar(&f.crawlDelay, "crawl-delay", 0, "override robots.txt Crawl-delay between page starts (0 = use robots.txt)")
|
||||
fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml")
|
||||
fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)")
|
||||
fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap <noscript> content instead of dropping it")
|
||||
@@ -102,6 +104,9 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid url %q: %w", arg, err)
|
||||
}
|
||||
if f.crawlDelay < 0 {
|
||||
return fmt.Errorf("--crawl-delay must be >= 0")
|
||||
}
|
||||
|
||||
cfg := clone.DefaultConfig()
|
||||
cfg.OutDir = f.out
|
||||
@@ -139,6 +144,7 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
|
||||
cfg.ScopePrefix = f.scopePrefix
|
||||
cfg.ExcludePaths = f.exclude
|
||||
cfg.RespectRobots = !f.noRobots
|
||||
cfg.CrawlDelay = f.crawlDelay
|
||||
cfg.FollowSitemap = !f.noSitemap
|
||||
cfg.Headless = !f.headful
|
||||
cfg.KeepNoscript = f.keepNoscript
|
||||
|
||||
+33
-3
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/tamnd/kage/sanitize"
|
||||
"github.com/tamnd/kage/urlx"
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
// Logf is an optional sink for human-readable progress lines.
|
||||
@@ -43,9 +44,12 @@ type Cloner struct {
|
||||
mu sync.Mutex
|
||||
seenAssets map[string]bool
|
||||
enqueued int // pages offered to the queue
|
||||
wg sync.WaitGroup
|
||||
pageJobs chan pageItem
|
||||
assetJobs chan assetItem
|
||||
|
||||
crawlLimiter *rate.Limiter
|
||||
|
||||
wg sync.WaitGroup
|
||||
pageJobs chan pageItem
|
||||
assetJobs chan assetItem
|
||||
|
||||
muContent sync.Mutex
|
||||
seenContent map[string]string // sha-256 of page bytes -> first path written
|
||||
@@ -144,6 +148,7 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) {
|
||||
defer func() { _ = c.pool.Close() }()
|
||||
|
||||
c.loadRobots(ctx)
|
||||
c.setupCrawlDelayLimiter()
|
||||
|
||||
// Start workers.
|
||||
var workers sync.WaitGroup
|
||||
@@ -218,6 +223,19 @@ func (c *Cloner) loadRobots(ctx context.Context) {
|
||||
c.robots = robots.Parse(string(data), "kage")
|
||||
}
|
||||
|
||||
func (c *Cloner) setupCrawlDelayLimiter() {
|
||||
delay := c.cfg.CrawlDelay
|
||||
if delay <= 0 && c.cfg.RespectRobots && c.robots != nil {
|
||||
delay = c.robots.CrawlDelay
|
||||
}
|
||||
if delay <= 0 {
|
||||
c.crawlLimiter = nil
|
||||
return
|
||||
}
|
||||
|
||||
c.crawlLimiter = rate.NewLimiter(rate.Every(delay), 1)
|
||||
}
|
||||
|
||||
// seedSitemaps adds in-scope sitemap URLs (from robots and the default path) to
|
||||
// the frontier.
|
||||
func (c *Cloner) seedSitemaps(ctx context.Context) {
|
||||
@@ -252,6 +270,9 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
|
||||
c.stats.skipped.Add(1)
|
||||
return
|
||||
}
|
||||
if !c.waitForCrawlDelay(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
res, err := c.pool.Render(ctx, j.u.String())
|
||||
if err != nil {
|
||||
@@ -324,6 +345,15 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
|
||||
c.stats.recordPage(c.pagePathKey(j.u), deduped)
|
||||
}
|
||||
|
||||
// waitForCrawlDelay spaces page render starts.
|
||||
func (c *Cloner) waitForCrawlDelay(ctx context.Context) bool {
|
||||
if c.crawlLimiter == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
return c.crawlLimiter.Wait(ctx) == nil
|
||||
}
|
||||
|
||||
// processAsset downloads one asset, rewriting CSS references on the way, and
|
||||
// writes it to its deterministic local path.
|
||||
func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/tamnd/kage/browser"
|
||||
"github.com/tamnd/kage/robots"
|
||||
"github.com/tamnd/kage/urlx"
|
||||
)
|
||||
|
||||
@@ -178,6 +179,56 @@ func TestPageKeyCollapsesDuplicates(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlDelaySpacesPageStarts(t *testing.T) {
|
||||
seed, _ := urlx.ParseSeed("https://ex.com")
|
||||
cfg := DefaultConfig()
|
||||
cfg.RespectRobots = true
|
||||
c := New(seed, cfg, nil)
|
||||
c.robots = &robots.Matcher{CrawlDelay: 20 * time.Millisecond}
|
||||
c.setupCrawlDelayLimiter()
|
||||
|
||||
ctx := context.Background()
|
||||
if !c.waitForCrawlDelay(ctx) {
|
||||
t.Fatal("first crawl-delay wait returned false")
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
if !c.waitForCrawlDelay(ctx) {
|
||||
t.Fatal("second crawl-delay wait returned false")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < 15*time.Millisecond {
|
||||
t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlDelayFlagOverridesRobots(t *testing.T) {
|
||||
seed, _ := urlx.ParseSeed("https://ex.com")
|
||||
cfg := DefaultConfig()
|
||||
cfg.RespectRobots = true
|
||||
cfg.CrawlDelay = 20 * time.Millisecond
|
||||
c := New(seed, cfg, nil)
|
||||
c.robots = &robots.Matcher{CrawlDelay: time.Minute}
|
||||
c.setupCrawlDelayLimiter()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
||||
defer cancel()
|
||||
if !c.waitForCrawlDelay(ctx) {
|
||||
t.Fatal("first crawl-delay wait returned false")
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
if !c.waitForCrawlDelay(ctx) {
|
||||
t.Fatal("second crawl-delay wait returned false")
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
if elapsed < 15*time.Millisecond {
|
||||
t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed)
|
||||
}
|
||||
if elapsed > 150*time.Millisecond {
|
||||
t.Fatalf("second crawl-delay wait = %v, override likely ignored", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func mustURL(t *testing.T, raw string) *url.URL {
|
||||
t.Helper()
|
||||
u, err := url.Parse(raw)
|
||||
|
||||
@@ -57,6 +57,7 @@ type Config struct {
|
||||
ExcludePaths []string
|
||||
|
||||
RespectRobots bool
|
||||
CrawlDelay time.Duration // override robots.txt Crawl-delay when > 0
|
||||
FollowSitemap bool
|
||||
Headless bool
|
||||
KeepNoscript bool
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: "Installation"
|
||||
description: "Install kage from Go, Homebrew, a release archive, a Linux package, or the container image, and point it at a browser."
|
||||
description: "Install kage from Go, Homebrew, Scoop, a release archive, a Linux package, or the container image, and point it at a browser."
|
||||
weight: 20
|
||||
---
|
||||
|
||||
@@ -12,10 +12,38 @@ kage is a single binary. Pick whichever channel suits you.
|
||||
go install github.com/tamnd/kage/cmd/kage@latest
|
||||
```
|
||||
|
||||
## Homebrew
|
||||
## Homebrew (macOS)
|
||||
|
||||
```bash
|
||||
brew install tamnd/tap/kage
|
||||
brew install --cask tamnd/tap/kage
|
||||
```
|
||||
|
||||
The cask installs the prebuilt macOS binary. On Linux, use the packages below or
|
||||
`go install`.
|
||||
|
||||
## Scoop (Windows)
|
||||
|
||||
```bash
|
||||
scoop bucket add tamnd https://github.com/tamnd/scoop-bucket
|
||||
scoop install kage
|
||||
```
|
||||
|
||||
## Linux (apt and dnf)
|
||||
|
||||
A signed apt and dnf repository tracks every release, so `apt upgrade` and
|
||||
`dnf upgrade` keep kage current.
|
||||
|
||||
```bash
|
||||
# Debian, Ubuntu
|
||||
curl -fsSL https://tamnd.github.io/linux-repo/gpg.key \
|
||||
| sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" \
|
||||
| sudo tee /etc/apt/sources.list.d/tamnd.list
|
||||
sudo apt update && sudo apt install kage
|
||||
|
||||
# Fedora, RHEL
|
||||
sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo
|
||||
sudo dnf install kage
|
||||
```
|
||||
|
||||
## Release archives and Linux packages
|
||||
@@ -24,14 +52,14 @@ Every [release](https://github.com/tamnd/kage/releases) attaches `tar.gz`
|
||||
archives (and a `.zip` for Windows) for Linux, macOS, Windows, and FreeBSD, plus
|
||||
`.deb`, `.rpm`, and `.apk` packages and a `checksums.txt` with a cosign
|
||||
signature. Download the one for your platform, extract `kage`, and put it on your
|
||||
`PATH`.
|
||||
`PATH`. To install a package directly without the repo above:
|
||||
|
||||
```bash
|
||||
# Debian/Ubuntu
|
||||
sudo dpkg -i kage_*_linux_amd64.deb
|
||||
sudo dpkg -i kage_*_amd64.deb
|
||||
|
||||
# Fedora/RHEL
|
||||
sudo rpm -i kage_*_linux_amd64.rpm
|
||||
sudo rpm -i kage-*.x86_64.rpm
|
||||
```
|
||||
|
||||
## Container
|
||||
|
||||
@@ -47,6 +47,7 @@ images, and fonts, and writes a browsable mirror to `<out>/<host>/`.
|
||||
| Flag | Default | Meaning |
|
||||
|------|---------|---------|
|
||||
| `--no-robots` | `false` | Ignore `robots.txt` |
|
||||
| `--crawl-delay` | `0s` | Override robots.txt `Crawl-delay` between page starts (0 = use robots.txt) |
|
||||
| `--no-sitemap` | `false` | Do not seed URLs from `sitemap.xml` |
|
||||
| `--user-agent` | Chrome UA | User-Agent for asset and robots fetches |
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ require (
|
||||
github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6
|
||||
golang.org/x/image v0.42.0
|
||||
golang.org/x/net v0.56.0
|
||||
golang.org/x/time v0.15.0
|
||||
)
|
||||
|
||||
require (
|
||||
|
||||
@@ -123,6 +123,8 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
|
||||
Reference in New Issue
Block a user