Dedup pages and assets by output path, add --refresh
Crawling keyed off the raw URL, so the same page reached over http and https, or as /index.html versus /, was a different frontier entry that nonetheless wrote to the same file. A clone of paulgraham.com did 948 render passes for 474 files. Key pages and assets by the local path they write instead, and collapse a directory-index document to its directory, so each page is fetched exactly once. Add --refresh to re-render a mirror in place (re-fetch every page, keep the directory, overwrite) and make --no-resume truly stateless by not persisting state.json. The default remains a resumable, idempotent crawl that skips work already on disk.
This commit is contained in:
Duc-Tam Nguyen
@@ -73,13 +73,21 @@ kage clone example.com --subdomains --scroll
|
||||
|
||||
# Resume an interrupted run (on by default; Ctrl-C saves state)
|
||||
kage clone example.com
|
||||
|
||||
# Re-render every page in place to pull in changed content
|
||||
kage clone example.com --refresh
|
||||
```
|
||||
|
||||
A clone is idempotent: each page is keyed by the file it writes, so the same URL
|
||||
reached over http and https, with or without a trailing slash, is fetched once.
|
||||
Re-running resumes where it left off; `--refresh` re-renders in place, `--force`
|
||||
wipes and starts clean.
|
||||
|
||||
Common flags:
|
||||
|
||||
| Flag | Default | Meaning |
|
||||
|------|---------|---------|
|
||||
| `-o, --out` | `kage-out` | Output root; the mirror lands in `<out>/<host>/` |
|
||||
| `-o, --out` | `$HOME/data/kage` | Output root; the mirror lands in `<out>/<host>/` |
|
||||
| `-p, --max-pages` | `0` | Stop after N pages (0 = unlimited) |
|
||||
| `-d, --max-depth` | `0` | Link-follow depth cap (0 = unlimited) |
|
||||
| `--scope-prefix` | | Only crawl pages whose path starts with this prefix |
|
||||
|
||||
+5
-1
@@ -39,6 +39,7 @@ type cloneFlags struct {
|
||||
chromeBin string
|
||||
controlURL string
|
||||
noResume bool
|
||||
refresh bool
|
||||
force bool
|
||||
quiet bool
|
||||
}
|
||||
@@ -57,7 +58,7 @@ func newCloneCmd() *cobra.Command {
|
||||
},
|
||||
}
|
||||
fs := cmd.Flags()
|
||||
fs.StringVarP(&f.out, "out", "o", "kage-out", "output root; the mirror lands in <out>/<host>/")
|
||||
fs.StringVarP(&f.out, "out", "o", clone.DefaultOutDir(), "output root; the mirror lands in <out>/<host>/")
|
||||
fs.StringVar(&f.reserved, "reserved", urlx.DefaultReserved, "reserved dir for assets and state")
|
||||
fs.IntVar(&f.workers, "workers", 4, "concurrent page render workers")
|
||||
fs.IntVar(&f.assetWorkers, "asset-workers", 8, "concurrent asset download workers")
|
||||
@@ -81,6 +82,7 @@ func newCloneCmd() *cobra.Command {
|
||||
fs.StringVar(&f.chromeBin, "chrome", "", "path to the Chrome/Chromium binary")
|
||||
fs.StringVar(&f.controlURL, "control-url", "", "attach to an existing Chrome DevTools endpoint")
|
||||
fs.BoolVar(&f.noResume, "no-resume", false, "do not reuse or write resume state")
|
||||
fs.BoolVar(&f.refresh, "refresh", false, "re-render every page in place to pull in changed content")
|
||||
fs.BoolVarP(&f.force, "force", "f", false, "delete any existing mirror for the host first")
|
||||
fs.BoolVarP(&f.quiet, "quiet", "q", false, "suppress per-page progress lines")
|
||||
return cmd
|
||||
@@ -117,6 +119,8 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
|
||||
cfg.ChromeBin = f.chromeBin
|
||||
cfg.ControlURL = f.controlURL
|
||||
cfg.Resume = !f.noResume
|
||||
cfg.Persist = !f.noResume
|
||||
cfg.Refresh = f.refresh
|
||||
cfg.Force = f.force
|
||||
|
||||
logf := func(format string, args ...any) {
|
||||
|
||||
+23
-6
@@ -84,13 +84,28 @@ func New(seed *url.URL, cfg Config, logf Logf) *Cloner {
|
||||
// Snapshot returns the current progress, for a CLI ticker.
|
||||
func (c *Cloner) Snapshot() Progress { return c.stats.snapshot() }
|
||||
|
||||
// pageKey is the dedup identity for a page: its output file. Two URLs that would
|
||||
// write to the same file (http vs https, "/" vs "/index.html", a trailing
|
||||
// slash) are the same page and must be crawled only once.
|
||||
func (c *Cloner) pageKey(u *url.URL) string {
|
||||
return urlx.LocalPath(c.seedHost, u, urlx.Page, c.cfg.Reserved)
|
||||
}
|
||||
|
||||
// assetKey is the dedup identity for an asset: its output file, so the same
|
||||
// bytes referenced over http and https download once.
|
||||
func (c *Cloner) assetKey(u *url.URL) string {
|
||||
return urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
|
||||
}
|
||||
|
||||
// Run executes the clone until the frontier drains, MaxPages is hit, or ctx is
|
||||
// cancelled (which flushes the resume state). It returns the final Result.
|
||||
func (c *Cloner) Run(ctx context.Context) (Result, error) {
|
||||
if c.cfg.Force {
|
||||
_ = os.RemoveAll(c.outRoot)
|
||||
}
|
||||
if c.cfg.Resume {
|
||||
// Refresh re-renders every page in place, so it deliberately skips loading
|
||||
// the prior visited set; everything else resumes from where it left off.
|
||||
if c.cfg.Resume && !c.cfg.Refresh {
|
||||
if err := c.front.load(c.statePth); err != nil {
|
||||
c.logf("resume: could not load state: %v", err)
|
||||
} else if n := c.front.visitedCount(); n > 0 {
|
||||
@@ -144,8 +159,10 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) {
|
||||
}()
|
||||
workers.Wait()
|
||||
|
||||
if err := c.front.save(c.statePth); err != nil {
|
||||
c.logf("could not save resume state: %v", err)
|
||||
if c.cfg.Persist {
|
||||
if err := c.front.save(c.statePth); err != nil {
|
||||
c.logf("could not save resume state: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
res := Result{Progress: c.stats.snapshot(), OutDir: c.outRoot}
|
||||
@@ -211,7 +228,7 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
key := urlx.Key(j.u)
|
||||
key := c.pageKey(j.u)
|
||||
if c.cfg.RespectRobots && !c.robots.Allowed(j.u.Path) {
|
||||
c.stats.skipped.Add(1)
|
||||
return
|
||||
@@ -308,7 +325,7 @@ func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool {
|
||||
if c.cfg.MaxDepth > 0 && depth > c.cfg.MaxDepth {
|
||||
return false
|
||||
}
|
||||
key := urlx.Key(u)
|
||||
key := c.pageKey(u)
|
||||
if c.front.isVisited(key) {
|
||||
return false
|
||||
}
|
||||
@@ -336,7 +353,7 @@ func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool {
|
||||
|
||||
// enqueueAsset offers an asset URL for download, deduping by canonical URL.
|
||||
func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) {
|
||||
key := urlx.Key(u)
|
||||
key := c.assetKey(u)
|
||||
c.mu.Lock()
|
||||
if c.seenAssets[key] {
|
||||
c.mu.Unlock()
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -151,6 +152,41 @@ func TestCloneEndToEnd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPageKeyCollapsesDuplicates guards the dedup identity: http vs https, a
|
||||
// trailing slash, and "/index.html" vs "/" all name the same output file, so a
|
||||
// page is crawled once rather than two-to-four times.
|
||||
func TestPageKeyCollapsesDuplicates(t *testing.T) {
|
||||
seed, _ := urlx.ParseSeed("https://ex.com")
|
||||
c := New(seed, DefaultConfig(), nil)
|
||||
|
||||
groups := [][]string{
|
||||
{"https://ex.com/", "http://ex.com/", "https://ex.com/index.html", "http://ex.com/index.html"},
|
||||
{"https://ex.com/docs", "http://ex.com/docs", "https://ex.com/docs/", "https://ex.com/docs/index.html"},
|
||||
}
|
||||
for _, g := range groups {
|
||||
want := c.pageKey(mustURL(t, g[0]))
|
||||
for _, raw := range g[1:] {
|
||||
if got := c.pageKey(mustURL(t, raw)); got != want {
|
||||
t.Errorf("pageKey(%q) = %q, want %q (same page)", raw, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Genuinely different pages must not collapse.
|
||||
if c.pageKey(mustURL(t, "https://ex.com/a")) == c.pageKey(mustURL(t, "https://ex.com/b")) {
|
||||
t.Error("distinct pages share a key")
|
||||
}
|
||||
}
|
||||
|
||||
func mustURL(t *testing.T, raw string) *url.URL {
|
||||
t.Helper()
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parse %q: %v", raw, err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
func TestCloneResumeSkipsVisited(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("resume test drives Chrome; skipped under -short")
|
||||
@@ -186,6 +222,45 @@ func TestCloneResumeSkipsVisited(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneRefreshReRenders(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("refresh test drives Chrome; skipped under -short")
|
||||
}
|
||||
if _, ok := browser.LookChrome(); !ok {
|
||||
t.Skip("no Chrome/Chromium found; skipping refresh test")
|
||||
}
|
||||
|
||||
srv := testSite(t)
|
||||
defer srv.Close()
|
||||
seed, _ := urlx.ParseSeed(srv.URL)
|
||||
|
||||
out := t.TempDir()
|
||||
cfg := DefaultConfig()
|
||||
cfg.OutDir = out
|
||||
cfg.Settle = 300 * time.Millisecond
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if _, err := New(seed, cfg, t.Logf).Run(ctx); err != nil {
|
||||
t.Fatalf("first run: %v", err)
|
||||
}
|
||||
|
||||
// Resume keeps everything in place but renders nothing new; refresh, on the
|
||||
// same mirror, re-renders every page so changed content is pulled back in.
|
||||
cfg.Refresh = true
|
||||
res, err := New(seed, cfg, t.Logf).Run(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("refresh run: %v", err)
|
||||
}
|
||||
if res.Pages < 2 {
|
||||
t.Fatalf("refresh should re-render every page, got %d", res.Pages)
|
||||
}
|
||||
if !fileExists(filepath.Join(out, seed.Hostname(), "index.html")) {
|
||||
t.Error("refresh removed the mirror instead of overwriting in place")
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
|
||||
+24
-3
@@ -4,12 +4,24 @@
|
||||
package clone
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/tamnd/kage/urlx"
|
||||
)
|
||||
|
||||
// DefaultOutDir is where mirrors land unless --out overrides it: a per-user data
|
||||
// directory ($HOME/data/kage) so clones from anywhere collect in one place,
|
||||
// falling back to a local kage-out when the home directory cannot be resolved.
|
||||
func DefaultOutDir() string {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil || home == "" {
|
||||
return "kage-out"
|
||||
}
|
||||
return filepath.Join(home, "data", "kage")
|
||||
}
|
||||
|
||||
// Config is the full set of knobs for a clone run. DefaultConfig fills the
|
||||
// baseline; the CLI overlays flags on top.
|
||||
type Config struct {
|
||||
@@ -41,8 +53,16 @@ type Config struct {
|
||||
ChromeBin string
|
||||
ControlURL string
|
||||
|
||||
Resume bool
|
||||
Force bool
|
||||
// Resume loads the prior run's visited set and skips pages already written,
|
||||
// so an interrupted or repeated clone picks up where it left off instead of
|
||||
// refetching. Refresh forces every page to be re-rendered in place (the
|
||||
// mirror is kept, files are overwritten) to pull in changed content. Force
|
||||
// deletes the mirror first for a clean-slate clone. Persist writes the
|
||||
// visited set back to state.json when the run ends.
|
||||
Resume bool
|
||||
Refresh bool
|
||||
Force bool
|
||||
Persist bool
|
||||
}
|
||||
|
||||
// DefaultUserAgent is a current desktop Chrome UA, used by the asset fetcher and
|
||||
@@ -53,7 +73,7 @@ const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +
|
||||
// DefaultConfig returns the baseline configuration.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
OutDir: "kage-out",
|
||||
OutDir: DefaultOutDir(),
|
||||
Reserved: urlx.DefaultReserved,
|
||||
Workers: 4,
|
||||
AssetWorkers: 8,
|
||||
@@ -68,6 +88,7 @@ func DefaultConfig() Config {
|
||||
FollowSitemap: true,
|
||||
Headless: true,
|
||||
Resume: true,
|
||||
Persist: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,9 +24,10 @@ images, and fonts, and writes a browsable mirror to `<out>/<host>/`.
|
||||
|
||||
| Flag | Default | Meaning |
|
||||
|------|---------|---------|
|
||||
| `-o, --out` | `kage-out` | Output root; the mirror lands in `<out>/<host>/` |
|
||||
| `-o, --out` | `$HOME/data/kage` | Output root; the mirror lands in `<out>/<host>/` |
|
||||
| `--reserved` | `_kage` | Reserved directory name for assets and crawl state |
|
||||
| `-f, --force` | `false` | Delete any existing mirror for the host before crawling |
|
||||
| `--refresh` | `false` | Re-render every page in place to pull in changed content |
|
||||
| `--no-resume` | `false` | Do not read or write resume state |
|
||||
|
||||
### Scope
|
||||
|
||||
@@ -20,7 +20,8 @@ kage's launcher can download a private copy of Chromium on first use.
|
||||
|
||||
## Output layout
|
||||
|
||||
A clone of `example.com` lands under `kage-out/example.com/`:
|
||||
A clone of `example.com` lands under `$HOME/data/kage/example.com/` (override the
|
||||
root with `-o/--out`):
|
||||
|
||||
```
|
||||
kage-out/example.com/
|
||||
@@ -50,5 +51,19 @@ Key points:
|
||||
`style.css?v=3` is saved with a short hash suffix so two versions never
|
||||
collide.
|
||||
- **State lives in the mirror.** `_kage/state.json` records every page written,
|
||||
which is what makes `--resume` able to skip completed work. Rename the reserved
|
||||
which is what lets a repeated run skip completed work. Rename the reserved
|
||||
directory with `--reserved` if `_kage` would clash with a real path on the site.
|
||||
|
||||
## Resume, refresh, and re-crawl
|
||||
|
||||
A clone is idempotent: every page is keyed by the file it writes, so the same
|
||||
page reached over `http` and `https`, with or without a trailing slash, or as
|
||||
`/index.html` versus `/`, is fetched exactly once. Re-running picks the work back
|
||||
up rather than starting over.
|
||||
|
||||
| You want to… | Use | What happens |
|
||||
|--------------|-----|--------------|
|
||||
| Continue an interrupted crawl | *(default)* | Loads `state.json`, skips pages already written, fetches only what is missing |
|
||||
| Pull in content that changed on the site | `--refresh` | Keeps the mirror, re-renders every page in place, overwrites with the new DOM |
|
||||
| Start completely clean | `--force` | Deletes the host's mirror, then crawls from scratch |
|
||||
| Run once and leave no trace | `--no-resume` | Skips nothing, writes no `state.json` |
|
||||
|
||||
+16
-1
@@ -206,7 +206,7 @@ func LocalPath(seedHost string, u *url.URL, kind Kind, reserved string) string {
|
||||
base = applyQuery(base, u)
|
||||
return joinClean(reserved, host, dir, base)
|
||||
default: // Page
|
||||
dir := strings.Trim(u.Path, "/")
|
||||
dir := collapseIndex(strings.Trim(u.Path, "/"))
|
||||
leaf := applyQuery("index.html", u)
|
||||
if strings.EqualFold(host, seedHost) {
|
||||
return joinClean(dir, leaf)
|
||||
@@ -217,6 +217,21 @@ func LocalPath(seedHost string, u *url.URL, kind Kind, reserved string) string {
|
||||
}
|
||||
}
|
||||
|
||||
// collapseIndex treats a directory-index document as the directory itself, so
|
||||
// "/" and "/index.html" map to one page, and "/docs/index.html" collapses to
|
||||
// "/docs". The argument is a path already trimmed of surrounding slashes.
|
||||
func collapseIndex(p string) string {
|
||||
for _, idx := range []string{"index.html", "index.htm"} {
|
||||
if p == idx {
|
||||
return ""
|
||||
}
|
||||
if rest, ok := strings.CutSuffix(p, "/"+idx); ok {
|
||||
return rest
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// splitAsset breaks an asset URL path into a directory and a filename, inventing
|
||||
// a filename for directory-like or empty paths.
|
||||
func splitAsset(u *url.URL) (dir, base string) {
|
||||
|
||||
@@ -139,6 +139,11 @@ func TestLocalPathPages(t *testing.T) {
|
||||
{"https://ex.com/docs/intro", "docs/intro/index.html"},
|
||||
{"https://ex.com/a.html", "a.html/index.html"},
|
||||
{"https://sub.ex.com/x", "sub.ex.com/x/index.html"},
|
||||
// A directory-index document is the directory itself, so it shares a
|
||||
// path with the bare directory and with the http/https variant.
|
||||
{"https://ex.com/index.html", "index.html"},
|
||||
{"http://ex.com/", "index.html"},
|
||||
{"https://ex.com/docs/index.html", "docs/index.html"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := LocalPath(seed, mustParse(t, c.u), Page, "")
|
||||
|
||||
Reference in New Issue
Block a user