Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1ba8c8943e | |||
| 35860d5aaa | |||
| 7483efd2a9 | |||
| 320b21a2ba | |||
| 2dabb93a78 | |||
| 8bc9c8bea1 | |||
| 28d08ade88 | |||
| 1e59c10e6f |
@@ -30,7 +30,7 @@ jobs:
|
|||||||
- uses: actions/checkout@v7.0.0
|
- uses: actions/checkout@v7.0.0
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
- uses: actions/setup-go@v6
|
- uses: actions/setup-go@v6.4.0
|
||||||
with:
|
with:
|
||||||
go-version-file: go.mod
|
go-version-file: go.mod
|
||||||
check-latest: true
|
check-latest: true
|
||||||
@@ -55,7 +55,7 @@ jobs:
|
|||||||
- uses: actions/checkout@v7.0.0
|
- uses: actions/checkout@v7.0.0
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
- uses: actions/setup-go@v6
|
- uses: actions/setup-go@v6.4.0
|
||||||
with:
|
with:
|
||||||
go-version-file: go.mod
|
go-version-file: go.mod
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|||||||
@@ -159,6 +159,15 @@ homebrew_casks:
|
|||||||
commit_author:
|
commit_author:
|
||||||
name: Duc-Tam Nguyen
|
name: Duc-Tam Nguyen
|
||||||
email: tamnd87@gmail.com
|
email: tamnd87@gmail.com
|
||||||
|
# Homebrew quarantines cask artifacts, and Gatekeeper kills quarantined
|
||||||
|
# binaries that are only ad-hoc signed (which cross-compiled Go binaries
|
||||||
|
# are). Strip the attribute at install so the binary runs first try.
|
||||||
|
hooks:
|
||||||
|
post:
|
||||||
|
install: |
|
||||||
|
if system_command("/usr/bin/xattr", args: ["-h"]).exit_status.zero?
|
||||||
|
system_command "/usr/bin/xattr", args: ["-dr", "com.apple.quarantine", "#{staged_path}/kage"]
|
||||||
|
end
|
||||||
|
|
||||||
scoops:
|
scoops:
|
||||||
# Scoop manifest for Windows, pushed to the bucket repository. It installs the
|
# Scoop manifest for Windows, pushed to the bucket repository. It installs the
|
||||||
|
|||||||
+11
-1
@@ -6,6 +6,15 @@ All notable changes to kage are recorded here. The format follows
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [0.3.9] - 2026-07-08
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- The Windows build no longer embeds the leakless watchdog binary that Windows Defender flags as `Trojan:Win32/Kepavll!rfn`, which made a fresh `scoop install` fail with a virus warning on `leakless.exe` ([#68](https://github.com/tamnd/kage/issues/68)).
|
||||||
|
go-rod's launcher imports [leakless](https://github.com/ysmood/leakless), which base64/gzip-embeds a prebuilt helper for every platform and links the Windows one into `kage.exe`.
|
||||||
|
kage already launches Chrome with leakless disabled, so the helper never ran, only added the flagged bytes.
|
||||||
|
A `replace` directive now points the package at an API-compatible stub under `third_party/leakless` that carries no embedded binary, dropping about 1.28 MB from the Windows build.
|
||||||
|
|
||||||
## [0.3.6] - 2026-06-19
|
## [0.3.6] - 2026-06-19
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
@@ -255,7 +264,8 @@ can browse offline, with every script stripped out.
|
|||||||
a multi-arch container image on GHCR (Chromium bundled), checksums, SBOMs, and
|
a multi-arch container image on GHCR (Chromium bundled), checksums, SBOMs, and
|
||||||
a cosign signature, all cut from one version tag by GoReleaser.
|
a cosign signature, all cut from one version tag by GoReleaser.
|
||||||
|
|
||||||
[Unreleased]: https://github.com/tamnd/kage/compare/v0.3.4...HEAD
|
[Unreleased]: https://github.com/tamnd/kage/compare/v0.3.9...HEAD
|
||||||
|
[0.3.9]: https://github.com/tamnd/kage/compare/v0.3.8...v0.3.9
|
||||||
[0.3.4]: https://github.com/tamnd/kage/compare/v0.3.3...v0.3.4
|
[0.3.4]: https://github.com/tamnd/kage/compare/v0.3.3...v0.3.4
|
||||||
[0.3.3]: https://github.com/tamnd/kage/compare/v0.3.2...v0.3.3
|
[0.3.3]: https://github.com/tamnd/kage/compare/v0.3.2...v0.3.3
|
||||||
[0.3.2]: https://github.com/tamnd/kage/compare/v0.3.1...v0.3.2
|
[0.3.2]: https://github.com/tamnd/kage/compare/v0.3.1...v0.3.2
|
||||||
|
|||||||
+13
-9
@@ -17,24 +17,28 @@ ARG TARGETPLATFORM
|
|||||||
# chromium for rendering; ca-certificates for HTTPS; tzdata for sane timestamps;
|
# chromium for rendering; ca-certificates for HTTPS; tzdata for sane timestamps;
|
||||||
# the font package so rendered pages have glyphs to lay out.
|
# the font package so rendered pages have glyphs to lay out.
|
||||||
RUN apk add --no-cache chromium ca-certificates tzdata font-noto \
|
RUN apk add --no-cache chromium ca-certificates tzdata font-noto \
|
||||||
&& adduser -D -H -u 10001 kage \
|
&& mkdir -p /out
|
||||||
&& mkdir -p /out \
|
|
||||||
&& chown kage:kage /out
|
|
||||||
|
|
||||||
COPY $TARGETPLATFORM/kage /usr/bin/kage
|
COPY $TARGETPLATFORM/kage /usr/bin/kage
|
||||||
|
|
||||||
USER kage
|
|
||||||
WORKDIR /out
|
WORKDIR /out
|
||||||
|
|
||||||
# Point kage at the bundled Chromium and write mirrors under /out by default:
|
# Point kage at the bundled Chromium and write mirrors under /out by default:
|
||||||
#
|
#
|
||||||
# docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com
|
# docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com
|
||||||
#
|
#
|
||||||
# The kage user has no home directory of its own, so HOME points at the mounted
|
# The container runs as root, and that is deliberate (issue #7). A bind-mounted
|
||||||
# /out volume. That keeps two things writable: kage's default output and resume
|
# /out is owned by whoever created it on the host, so only root can reliably
|
||||||
# state (it lands under $HOME/data/kage), and Chrome's profile and crash
|
# write into it; a fixed non-root uid cannot, and both kage's output and resume
|
||||||
# database. Without this both fail with a permission error in the container
|
# state (under $HOME/data/kage) then fail with "mkdir /out: permission denied".
|
||||||
# (issue #7), and the mounted volume captures nothing.
|
# The same unwritable HOME also breaks Chrome: it launches chrome_crashpad_handler
|
||||||
|
# with an empty crash database path, which aborts the whole browser with
|
||||||
|
# "chrome_crashpad_handler: --database is required" and fails every render.
|
||||||
|
# Running as root keeps /out and HOME writable whatever the host owns, so the
|
||||||
|
# one-liner above just works. This costs nothing in the sandbox: Chrome's sandbox
|
||||||
|
# is already off inside any container (kage drops it on container detection), so
|
||||||
|
# root here does not loosen a boundary that was holding. HOME points at /out so
|
||||||
|
# the default output and Chrome's writable state both land in the mounted volume.
|
||||||
ENV KAGE_CHROME=/usr/bin/chromium-browser \
|
ENV KAGE_CHROME=/usr/bin/chromium-browser \
|
||||||
HOME=/out
|
HOME=/out
|
||||||
|
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ Prefer a prebuilt binary? Grab an archive, a `.deb`/`.rpm`/`.apk`, or a checksum
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Homebrew (macOS)
|
# Homebrew (macOS)
|
||||||
brew install tamnd/tap/kage
|
brew install --cask tamnd/tap/kage
|
||||||
|
|
||||||
# Scoop (Windows)
|
# Scoop (Windows)
|
||||||
scoop bucket add tamnd https://github.com/tamnd/scoop-bucket
|
scoop bucket add tamnd https://github.com/tamnd/scoop-bucket
|
||||||
@@ -122,6 +122,7 @@ The flags you'll actually reach for:
|
|||||||
| `--scroll` | `false` | Auto-scroll each page to trigger lazy loading |
|
| `--scroll` | `false` | Auto-scroll each page to trigger lazy loading |
|
||||||
| `--workers` | `4` | How many pages to render at once |
|
| `--workers` | `4` | How many pages to render at once |
|
||||||
| `--no-robots` | `false` | Ignore `robots.txt` (be nice) |
|
| `--no-robots` | `false` | Ignore `robots.txt` (be nice) |
|
||||||
|
| `--crawl-delay` | `0s` | Override robots.txt `Crawl-delay` between page starts |
|
||||||
| `-f, --force` | `false` | Delete any existing mirror for the host first |
|
| `-f, --force` | `false` | Delete any existing mirror for the host first |
|
||||||
| `--chrome` | | Path to the Chrome/Chromium binary |
|
| `--chrome` | | Path to the Chrome/Chromium binary |
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
package browser
|
||||||
|
|
||||||
|
import "runtime"
|
||||||
|
|
||||||
|
func launcherLeakless() bool {
|
||||||
|
return runtime.GOOS != "windows"
|
||||||
|
}
|
||||||
+8
-7
@@ -165,7 +165,7 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
|
|||||||
|
|
||||||
controlURL := p.opts.ControlURL
|
controlURL := p.opts.ControlURL
|
||||||
if controlURL == "" {
|
if controlURL == "" {
|
||||||
l := launcher.New().
|
l := launcher.New().Leakless(launcherLeakless()).
|
||||||
Headless(p.opts.Headless).
|
Headless(p.opts.Headless).
|
||||||
Set("disable-blink-features", "AutomationControlled").
|
Set("disable-blink-features", "AutomationControlled").
|
||||||
Set("disable-gpu", "")
|
Set("disable-gpu", "")
|
||||||
@@ -183,13 +183,14 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
|
|||||||
// In a container, the default /dev/shm is only 64 MB, too small for
|
// In a container, the default /dev/shm is only 64 MB, too small for
|
||||||
// Chrome's renderer on large pages, so steer it to a temp file instead.
|
// Chrome's renderer on large pages, so steer it to a temp file instead.
|
||||||
// Outside a container /dev/shm is roomy and faster, so leave it alone.
|
// Outside a container /dev/shm is roomy and faster, so leave it alone.
|
||||||
// Chrome's crashpad handler also aborts with "--database is required" in a
|
//
|
||||||
// minimal container, which fails the whole launch (issue #7), so turn the
|
// The "chrome_crashpad_handler: --database is required" abort seen in
|
||||||
// crash reporter off there. kage never uploads Chrome crash dumps anyway.
|
// containers (issue #7) is not fixed here: the crash-reporter flags do not
|
||||||
|
// stop Chrome from spawning the handler. Its real cause is an unwritable
|
||||||
|
// HOME, which leaves the crash database path empty; the image keeps HOME
|
||||||
|
// writable instead (see the Dockerfile).
|
||||||
if inContainer() {
|
if inContainer() {
|
||||||
l = l.Set("disable-dev-shm-usage", "").
|
l = l.Set("disable-dev-shm-usage", "")
|
||||||
Set("disable-crash-reporter", "").
|
|
||||||
Set("disable-breakpad", "")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if bin := p.chromeBin(); bin != "" {
|
if bin := p.chromeBin(); bin != "" {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -83,6 +84,14 @@ func TestDisableSandboxContainer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLauncherLeaklessDisabledOnWindows(t *testing.T) {
|
||||||
|
got := launcherLeakless()
|
||||||
|
want := runtime.GOOS != "windows"
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("launcherLeakless() = %v on %s; want %v", got, runtime.GOOS, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRenderCapturesFinalDOM(t *testing.T) {
|
func TestRenderCapturesFinalDOM(t *testing.T) {
|
||||||
if testing.Short() {
|
if testing.Short() {
|
||||||
t.Skip("render test drives Chrome; skipped under -short")
|
t.Skip("render test drives Chrome; skipped under -short")
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ type cloneFlags struct {
|
|||||||
scopePrefix string
|
scopePrefix string
|
||||||
exclude []string
|
exclude []string
|
||||||
noRobots bool
|
noRobots bool
|
||||||
|
crawlDelay time.Duration
|
||||||
noSitemap bool
|
noSitemap bool
|
||||||
headful bool
|
headful bool
|
||||||
keepNoscript bool
|
keepNoscript bool
|
||||||
@@ -84,6 +85,7 @@ func newCloneCmd() *cobra.Command {
|
|||||||
fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix")
|
fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix")
|
||||||
fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)")
|
fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)")
|
||||||
fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)")
|
fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)")
|
||||||
|
fs.DurationVar(&f.crawlDelay, "crawl-delay", 0, "override robots.txt Crawl-delay between page starts (0 = use robots.txt)")
|
||||||
fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml")
|
fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml")
|
||||||
fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)")
|
fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)")
|
||||||
fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap <noscript> content instead of dropping it")
|
fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap <noscript> content instead of dropping it")
|
||||||
@@ -102,6 +104,9 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("invalid url %q: %w", arg, err)
|
return fmt.Errorf("invalid url %q: %w", arg, err)
|
||||||
}
|
}
|
||||||
|
if f.crawlDelay < 0 {
|
||||||
|
return fmt.Errorf("--crawl-delay must be >= 0")
|
||||||
|
}
|
||||||
|
|
||||||
cfg := clone.DefaultConfig()
|
cfg := clone.DefaultConfig()
|
||||||
cfg.OutDir = f.out
|
cfg.OutDir = f.out
|
||||||
@@ -139,6 +144,7 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
|
|||||||
cfg.ScopePrefix = f.scopePrefix
|
cfg.ScopePrefix = f.scopePrefix
|
||||||
cfg.ExcludePaths = f.exclude
|
cfg.ExcludePaths = f.exclude
|
||||||
cfg.RespectRobots = !f.noRobots
|
cfg.RespectRobots = !f.noRobots
|
||||||
|
cfg.CrawlDelay = f.crawlDelay
|
||||||
cfg.FollowSitemap = !f.noSitemap
|
cfg.FollowSitemap = !f.noSitemap
|
||||||
cfg.Headless = !f.headful
|
cfg.Headless = !f.headful
|
||||||
cfg.KeepNoscript = f.keepNoscript
|
cfg.KeepNoscript = f.keepNoscript
|
||||||
|
|||||||
+33
-3
@@ -19,6 +19,7 @@ import (
|
|||||||
"github.com/tamnd/kage/sanitize"
|
"github.com/tamnd/kage/sanitize"
|
||||||
"github.com/tamnd/kage/urlx"
|
"github.com/tamnd/kage/urlx"
|
||||||
"golang.org/x/net/html"
|
"golang.org/x/net/html"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Logf is an optional sink for human-readable progress lines.
|
// Logf is an optional sink for human-readable progress lines.
|
||||||
@@ -43,9 +44,12 @@ type Cloner struct {
|
|||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
seenAssets map[string]bool
|
seenAssets map[string]bool
|
||||||
enqueued int // pages offered to the queue
|
enqueued int // pages offered to the queue
|
||||||
wg sync.WaitGroup
|
|
||||||
pageJobs chan pageItem
|
crawlLimiter *rate.Limiter
|
||||||
assetJobs chan assetItem
|
|
||||||
|
wg sync.WaitGroup
|
||||||
|
pageJobs chan pageItem
|
||||||
|
assetJobs chan assetItem
|
||||||
|
|
||||||
muContent sync.Mutex
|
muContent sync.Mutex
|
||||||
seenContent map[string]string // sha-256 of page bytes -> first path written
|
seenContent map[string]string // sha-256 of page bytes -> first path written
|
||||||
@@ -144,6 +148,7 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) {
|
|||||||
defer func() { _ = c.pool.Close() }()
|
defer func() { _ = c.pool.Close() }()
|
||||||
|
|
||||||
c.loadRobots(ctx)
|
c.loadRobots(ctx)
|
||||||
|
c.setupCrawlDelayLimiter()
|
||||||
|
|
||||||
// Start workers.
|
// Start workers.
|
||||||
var workers sync.WaitGroup
|
var workers sync.WaitGroup
|
||||||
@@ -218,6 +223,19 @@ func (c *Cloner) loadRobots(ctx context.Context) {
|
|||||||
c.robots = robots.Parse(string(data), "kage")
|
c.robots = robots.Parse(string(data), "kage")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Cloner) setupCrawlDelayLimiter() {
|
||||||
|
delay := c.cfg.CrawlDelay
|
||||||
|
if delay <= 0 && c.cfg.RespectRobots && c.robots != nil {
|
||||||
|
delay = c.robots.CrawlDelay
|
||||||
|
}
|
||||||
|
if delay <= 0 {
|
||||||
|
c.crawlLimiter = nil
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.crawlLimiter = rate.NewLimiter(rate.Every(delay), 1)
|
||||||
|
}
|
||||||
|
|
||||||
// seedSitemaps adds in-scope sitemap URLs (from robots and the default path) to
|
// seedSitemaps adds in-scope sitemap URLs (from robots and the default path) to
|
||||||
// the frontier.
|
// the frontier.
|
||||||
func (c *Cloner) seedSitemaps(ctx context.Context) {
|
func (c *Cloner) seedSitemaps(ctx context.Context) {
|
||||||
@@ -252,6 +270,9 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
|
|||||||
c.stats.skipped.Add(1)
|
c.stats.skipped.Add(1)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if !c.waitForCrawlDelay(ctx) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
res, err := c.pool.Render(ctx, j.u.String())
|
res, err := c.pool.Render(ctx, j.u.String())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -324,6 +345,15 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
|
|||||||
c.stats.recordPage(c.pagePathKey(j.u), deduped)
|
c.stats.recordPage(c.pagePathKey(j.u), deduped)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// waitForCrawlDelay spaces page render starts.
|
||||||
|
func (c *Cloner) waitForCrawlDelay(ctx context.Context) bool {
|
||||||
|
if c.crawlLimiter == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.crawlLimiter.Wait(ctx) == nil
|
||||||
|
}
|
||||||
|
|
||||||
// processAsset downloads one asset, rewriting CSS references on the way, and
|
// processAsset downloads one asset, rewriting CSS references on the way, and
|
||||||
// writes it to its deterministic local path.
|
// writes it to its deterministic local path.
|
||||||
func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
|
func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/tamnd/kage/browser"
|
"github.com/tamnd/kage/browser"
|
||||||
|
"github.com/tamnd/kage/robots"
|
||||||
"github.com/tamnd/kage/urlx"
|
"github.com/tamnd/kage/urlx"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -178,6 +179,56 @@ func TestPageKeyCollapsesDuplicates(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCrawlDelaySpacesPageStarts(t *testing.T) {
|
||||||
|
seed, _ := urlx.ParseSeed("https://ex.com")
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.RespectRobots = true
|
||||||
|
c := New(seed, cfg, nil)
|
||||||
|
c.robots = &robots.Matcher{CrawlDelay: 20 * time.Millisecond}
|
||||||
|
c.setupCrawlDelayLimiter()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
if !c.waitForCrawlDelay(ctx) {
|
||||||
|
t.Fatal("first crawl-delay wait returned false")
|
||||||
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
if !c.waitForCrawlDelay(ctx) {
|
||||||
|
t.Fatal("second crawl-delay wait returned false")
|
||||||
|
}
|
||||||
|
if elapsed := time.Since(start); elapsed < 15*time.Millisecond {
|
||||||
|
t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCrawlDelayFlagOverridesRobots(t *testing.T) {
|
||||||
|
seed, _ := urlx.ParseSeed("https://ex.com")
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.RespectRobots = true
|
||||||
|
cfg.CrawlDelay = 20 * time.Millisecond
|
||||||
|
c := New(seed, cfg, nil)
|
||||||
|
c.robots = &robots.Matcher{CrawlDelay: time.Minute}
|
||||||
|
c.setupCrawlDelayLimiter()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
if !c.waitForCrawlDelay(ctx) {
|
||||||
|
t.Fatal("first crawl-delay wait returned false")
|
||||||
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
if !c.waitForCrawlDelay(ctx) {
|
||||||
|
t.Fatal("second crawl-delay wait returned false")
|
||||||
|
}
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
if elapsed < 15*time.Millisecond {
|
||||||
|
t.Fatalf("second crawl-delay wait = %v, want at least 15ms", elapsed)
|
||||||
|
}
|
||||||
|
if elapsed > 150*time.Millisecond {
|
||||||
|
t.Fatalf("second crawl-delay wait = %v, override likely ignored", elapsed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func mustURL(t *testing.T, raw string) *url.URL {
|
func mustURL(t *testing.T, raw string) *url.URL {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
u, err := url.Parse(raw)
|
u, err := url.Parse(raw)
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ type Config struct {
|
|||||||
ExcludePaths []string
|
ExcludePaths []string
|
||||||
|
|
||||||
RespectRobots bool
|
RespectRobots bool
|
||||||
|
CrawlDelay time.Duration // override robots.txt Crawl-delay when > 0
|
||||||
FollowSitemap bool
|
FollowSitemap bool
|
||||||
Headless bool
|
Headless bool
|
||||||
KeepNoscript bool
|
KeepNoscript bool
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ go install github.com/tamnd/kage/cmd/kage@latest
|
|||||||
## Homebrew (macOS)
|
## Homebrew (macOS)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
brew install tamnd/tap/kage
|
brew install --cask tamnd/tap/kage
|
||||||
```
|
```
|
||||||
|
|
||||||
The cask installs the prebuilt macOS binary. On Linux, use the packages below or
|
The cask installs the prebuilt macOS binary. On Linux, use the packages below or
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ images, and fonts, and writes a browsable mirror to `<out>/<host>/`.
|
|||||||
| Flag | Default | Meaning |
|
| Flag | Default | Meaning |
|
||||||
|------|---------|---------|
|
|------|---------|---------|
|
||||||
| `--no-robots` | `false` | Ignore `robots.txt` |
|
| `--no-robots` | `false` | Ignore `robots.txt` |
|
||||||
|
| `--crawl-delay` | `0s` | Override robots.txt `Crawl-delay` between page starts (0 = use robots.txt) |
|
||||||
| `--no-sitemap` | `false` | Do not seed URLs from `sitemap.xml` |
|
| `--no-sitemap` | `false` | Do not seed URLs from `sitemap.xml` |
|
||||||
| `--user-agent` | Chrome UA | User-Agent for asset and robots fetches |
|
| `--user-agent` | Chrome UA | User-Agent for asset and robots fetches |
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,12 @@ weight: 40
|
|||||||
|
|
||||||
The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version.
|
The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version.
|
||||||
|
|
||||||
|
## v0.3.9
|
||||||
|
|
||||||
|
A fix for the antivirus warning some Windows users hit when installing kage.
|
||||||
|
|
||||||
|
- **The Windows build no longer ships the leakless helper antivirus flags.** kage renders pages with [go-rod](https://github.com/go-rod/rod), whose launcher pulls in [leakless](https://github.com/ysmood/leakless), a small watchdog that force-kills Chrome if kage exits. leakless carries a prebuilt helper binary for every platform and links the Windows one straight into `kage.exe`. Windows Defender recognises that helper as `Trojan:Win32/Kepavll!rfn` and quarantines it, so a fresh `scoop install` failed with a virus warning on `leakless.exe` ([#68](https://github.com/tamnd/kage/issues/68)). kage already launches Chrome with leakless switched off, so the helper never ran anyway. It is now replaced with a stub that carries no embedded binary, which drops about 1.28 MB from the Windows build and clears the warning. Thanks to John Pywtorak for the report. `go install`, unaffected before, stays clean.
|
||||||
|
|
||||||
## v0.3.4
|
## v0.3.4
|
||||||
|
|
||||||
Two community fixes: a clean stop for `kage serve`, and pages with heavy JavaScript that used to be dropped.
|
Two community fixes: a clean stop for `kage serve`, and pages with heavy JavaScript that used to be dropped.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
module github.com/tamnd/kage
|
module github.com/tamnd/kage
|
||||||
|
|
||||||
go 1.26.4
|
go 1.26.5
|
||||||
|
|
||||||
require (
|
require (
|
||||||
charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410
|
charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410
|
||||||
@@ -14,6 +14,7 @@ require (
|
|||||||
github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6
|
github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6
|
||||||
golang.org/x/image v0.42.0
|
golang.org/x/image v0.42.0
|
||||||
golang.org/x/net v0.56.0
|
golang.org/x/net v0.56.0
|
||||||
|
golang.org/x/time v0.15.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@@ -53,3 +54,11 @@ require (
|
|||||||
golang.org/x/text v0.38.0 // indirect
|
golang.org/x/text v0.38.0 // indirect
|
||||||
google.golang.org/protobuf v1.34.2 // indirect
|
google.golang.org/protobuf v1.34.2 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// go-rod's launcher imports github.com/ysmood/leakless, which base64/gzip-embeds
|
||||||
|
// a prebuilt leakless.exe into the Windows build. Antivirus engines flag that
|
||||||
|
// embedded helper as malware and quarantine kage on install (issue #68). kage
|
||||||
|
// always launches Chrome with leakless disabled (browser/leakless.go), so the
|
||||||
|
// guard is dead weight; this replace swaps in an API-compatible stub that
|
||||||
|
// carries no embedded binary.
|
||||||
|
replace github.com/ysmood/leakless => ./third_party/leakless
|
||||||
|
|||||||
@@ -107,9 +107,6 @@ github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
|
|||||||
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
|
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
|
||||||
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
|
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
|
||||||
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
|
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
|
||||||
github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
|
||||||
github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
|
|
||||||
github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
|
||||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||||
@@ -123,6 +120,8 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
|||||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||||
|
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||||
|
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||||
|
|||||||
Vendored
+3
@@ -0,0 +1,3 @@
|
|||||||
|
module github.com/ysmood/leakless
|
||||||
|
|
||||||
|
go 1.26.4
|
||||||
Vendored
+55
@@ -0,0 +1,55 @@
|
|||||||
|
// Package leakless is kage's drop-in replacement for
|
||||||
|
// github.com/ysmood/leakless, wired in through a replace directive in the root
|
||||||
|
// go.mod.
|
||||||
|
//
|
||||||
|
// The upstream package guards a child process by extracting a small helper
|
||||||
|
// executable that force-kills the child when the parent dies. It ships that
|
||||||
|
// helper by base64/gzip-embedding a prebuilt binary for every target
|
||||||
|
// (bin_amd64_windows.go and friends), so the packed leakless.exe ends up linked
|
||||||
|
// into any program that imports the package, kage included. Antivirus engines
|
||||||
|
// flag that embedded Windows helper as malware, so a fresh install of kage got
|
||||||
|
// quarantined before it ever ran (issue #68).
|
||||||
|
//
|
||||||
|
// kage already launches Chrome with leakless disabled (see
|
||||||
|
// browser/leakless.go), so the guard is never used. This stub keeps the exact
|
||||||
|
// public surface go-rod's launcher depends on (New, Support, LockPort, and the
|
||||||
|
// Launcher type's Command/Pid/Err) while carrying no embedded binary, which
|
||||||
|
// removes the false positive entirely. Support reports no guard is available,
|
||||||
|
// so go-rod's launcher never takes the leakless path even if a caller asked
|
||||||
|
// for it.
|
||||||
|
package leakless
|
||||||
|
|
||||||
|
import "os/exec"
|
||||||
|
|
||||||
|
// Launcher mirrors the upstream type. The channel is left unbuffered and is
|
||||||
|
// never written to, matching the "may never receive the pid" contract go-rod
|
||||||
|
// already tolerates.
|
||||||
|
type Launcher struct {
|
||||||
|
pid chan int
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns a Launcher. It allocates nothing beyond the pid channel.
|
||||||
|
func New() *Launcher {
|
||||||
|
return &Launcher{pid: make(chan int)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Command builds the command without a guard wrapper. Because Support returns
|
||||||
|
// false, go-rod never calls this in practice; if some other caller did, running
|
||||||
|
// the target directly is the correct no-guard behaviour.
|
||||||
|
func (l *Launcher) Command(name string, arg ...string) *exec.Cmd {
|
||||||
|
return exec.Command(name, arg...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pid returns the (never-signalled) pid channel.
|
||||||
|
func (l *Launcher) Pid() chan int { return l.pid }
|
||||||
|
|
||||||
|
// Err returns the guard error, always empty here since there is no guard.
|
||||||
|
func (l *Launcher) Err() string { return "" }
|
||||||
|
|
||||||
|
// Support reports whether a guard binary is available. It always returns false
|
||||||
|
// so callers skip leakless entirely.
|
||||||
|
func Support() bool { return false }
|
||||||
|
|
||||||
|
// LockPort is the cross-process mutex the upstream guard uses to serialise
|
||||||
|
// extraction. With no guard there is nothing to serialise, so it is a no-op.
|
||||||
|
func LockPort(port int) func() { return func() {} }
|
||||||
Reference in New Issue
Block a user