From a871265cbe862d762a684c57be9e22220eaf5cf4 Mon Sep 17 00:00:00 2001 From: Duc-Tam Nguyen Date: Sun, 14 Jun 2026 19:06:28 +0700 Subject: [PATCH] Dedup pages and assets by output path, add --refresh Crawling keyed off the raw URL, so the same page reached over http and https, or as /index.html versus /, was a different frontier entry that nonetheless wrote to the same file. A clone of paulgraham.com did 948 render passes for 474 files. Key pages and assets by the local path they write instead, and collapse a directory-index document to its directory, so each page is fetched exactly once. Add --refresh to re-render a mirror in place (re-fetch every page, keep the directory, overwrite) and make --no-resume truly stateless by not persisting state.json. The default remains a resumable, idempotent crawl that skips work already on disk. --- README.md | 10 +++- cli/clone.go | 6 +- clone/cloner.go | 29 ++++++++-- clone/cloner_test.go | 75 +++++++++++++++++++++++++ clone/config.go | 27 ++++++++- docs/content/reference/cli.md | 3 +- docs/content/reference/configuration.md | 19 ++++++- urlx/urlx.go | 17 +++++- urlx/urlx_test.go | 5 ++ 9 files changed, 176 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 4f02fe2..545c2e5 100644 --- a/README.md +++ b/README.md @@ -73,13 +73,21 @@ kage clone example.com --subdomains --scroll # Resume an interrupted run (on by default; Ctrl-C saves state) kage clone example.com + +# Re-render every page in place to pull in changed content +kage clone example.com --refresh ``` +A clone is idempotent: each page is keyed by the file it writes, so the same URL +reached over http and https, with or without a trailing slash, is fetched once. +Re-running resumes where it left off; `--refresh` re-renders in place, `--force` +wipes and starts clean. + Common flags: | Flag | Default | Meaning | |------|---------|---------| -| `-o, --out` | `kage-out` | Output root; the mirror lands in `//` | +| `-o, --out` | `$HOME/data/kage` | Output root; the mirror lands in `//` | | `-p, --max-pages` | `0` | Stop after N pages (0 = unlimited) | | `-d, --max-depth` | `0` | Link-follow depth cap (0 = unlimited) | | `--scope-prefix` | | Only crawl pages whose path starts with this prefix | diff --git a/cli/clone.go b/cli/clone.go index 02a7bf0..7d221dc 100644 --- a/cli/clone.go +++ b/cli/clone.go @@ -39,6 +39,7 @@ type cloneFlags struct { chromeBin string controlURL string noResume bool + refresh bool force bool quiet bool } @@ -57,7 +58,7 @@ func newCloneCmd() *cobra.Command { }, } fs := cmd.Flags() - fs.StringVarP(&f.out, "out", "o", "kage-out", "output root; the mirror lands in //") + fs.StringVarP(&f.out, "out", "o", clone.DefaultOutDir(), "output root; the mirror lands in //") fs.StringVar(&f.reserved, "reserved", urlx.DefaultReserved, "reserved dir for assets and state") fs.IntVar(&f.workers, "workers", 4, "concurrent page render workers") fs.IntVar(&f.assetWorkers, "asset-workers", 8, "concurrent asset download workers") @@ -81,6 +82,7 @@ func newCloneCmd() *cobra.Command { fs.StringVar(&f.chromeBin, "chrome", "", "path to the Chrome/Chromium binary") fs.StringVar(&f.controlURL, "control-url", "", "attach to an existing Chrome DevTools endpoint") fs.BoolVar(&f.noResume, "no-resume", false, "do not reuse or write resume state") + fs.BoolVar(&f.refresh, "refresh", false, "re-render every page in place to pull in changed content") fs.BoolVarP(&f.force, "force", "f", false, "delete any existing mirror for the host first") fs.BoolVarP(&f.quiet, "quiet", "q", false, "suppress per-page progress lines") return cmd @@ -117,6 +119,8 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error { cfg.ChromeBin = f.chromeBin cfg.ControlURL = f.controlURL cfg.Resume = !f.noResume + cfg.Persist = !f.noResume + cfg.Refresh = f.refresh cfg.Force = f.force logf := func(format string, args ...any) { diff --git a/clone/cloner.go b/clone/cloner.go index df69867..ce4da92 100644 --- a/clone/cloner.go +++ b/clone/cloner.go @@ -84,13 +84,28 @@ func New(seed *url.URL, cfg Config, logf Logf) *Cloner { // Snapshot returns the current progress, for a CLI ticker. func (c *Cloner) Snapshot() Progress { return c.stats.snapshot() } +// pageKey is the dedup identity for a page: its output file. Two URLs that would +// write to the same file (http vs https, "/" vs "/index.html", a trailing +// slash) are the same page and must be crawled only once. +func (c *Cloner) pageKey(u *url.URL) string { + return urlx.LocalPath(c.seedHost, u, urlx.Page, c.cfg.Reserved) +} + +// assetKey is the dedup identity for an asset: its output file, so the same +// bytes referenced over http and https download once. +func (c *Cloner) assetKey(u *url.URL) string { + return urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved) +} + // Run executes the clone until the frontier drains, MaxPages is hit, or ctx is // cancelled (which flushes the resume state). It returns the final Result. func (c *Cloner) Run(ctx context.Context) (Result, error) { if c.cfg.Force { _ = os.RemoveAll(c.outRoot) } - if c.cfg.Resume { + // Refresh re-renders every page in place, so it deliberately skips loading + // the prior visited set; everything else resumes from where it left off. + if c.cfg.Resume && !c.cfg.Refresh { if err := c.front.load(c.statePth); err != nil { c.logf("resume: could not load state: %v", err) } else if n := c.front.visitedCount(); n > 0 { @@ -144,8 +159,10 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) { }() workers.Wait() - if err := c.front.save(c.statePth); err != nil { - c.logf("could not save resume state: %v", err) + if c.cfg.Persist { + if err := c.front.save(c.statePth); err != nil { + c.logf("could not save resume state: %v", err) + } } res := Result{Progress: c.stats.snapshot(), OutDir: c.outRoot} @@ -211,7 +228,7 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) { if ctx.Err() != nil { return } - key := urlx.Key(j.u) + key := c.pageKey(j.u) if c.cfg.RespectRobots && !c.robots.Allowed(j.u.Path) { c.stats.skipped.Add(1) return @@ -308,7 +325,7 @@ func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool { if c.cfg.MaxDepth > 0 && depth > c.cfg.MaxDepth { return false } - key := urlx.Key(u) + key := c.pageKey(u) if c.front.isVisited(key) { return false } @@ -336,7 +353,7 @@ func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool { // enqueueAsset offers an asset URL for download, deduping by canonical URL. func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) { - key := urlx.Key(u) + key := c.assetKey(u) c.mu.Lock() if c.seenAssets[key] { c.mu.Unlock() diff --git a/clone/cloner_test.go b/clone/cloner_test.go index ee03bba..24d3ef6 100644 --- a/clone/cloner_test.go +++ b/clone/cloner_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "strings" @@ -151,6 +152,41 @@ func TestCloneEndToEnd(t *testing.T) { } } +// TestPageKeyCollapsesDuplicates guards the dedup identity: http vs https, a +// trailing slash, and "/index.html" vs "/" all name the same output file, so a +// page is crawled once rather than two-to-four times. +func TestPageKeyCollapsesDuplicates(t *testing.T) { + seed, _ := urlx.ParseSeed("https://ex.com") + c := New(seed, DefaultConfig(), nil) + + groups := [][]string{ + {"https://ex.com/", "http://ex.com/", "https://ex.com/index.html", "http://ex.com/index.html"}, + {"https://ex.com/docs", "http://ex.com/docs", "https://ex.com/docs/", "https://ex.com/docs/index.html"}, + } + for _, g := range groups { + want := c.pageKey(mustURL(t, g[0])) + for _, raw := range g[1:] { + if got := c.pageKey(mustURL(t, raw)); got != want { + t.Errorf("pageKey(%q) = %q, want %q (same page)", raw, got, want) + } + } + } + + // Genuinely different pages must not collapse. + if c.pageKey(mustURL(t, "https://ex.com/a")) == c.pageKey(mustURL(t, "https://ex.com/b")) { + t.Error("distinct pages share a key") + } +} + +func mustURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse %q: %v", raw, err) + } + return u +} + func TestCloneResumeSkipsVisited(t *testing.T) { if testing.Short() { t.Skip("resume test drives Chrome; skipped under -short") @@ -186,6 +222,45 @@ func TestCloneResumeSkipsVisited(t *testing.T) { } } +func TestCloneRefreshReRenders(t *testing.T) { + if testing.Short() { + t.Skip("refresh test drives Chrome; skipped under -short") + } + if _, ok := browser.LookChrome(); !ok { + t.Skip("no Chrome/Chromium found; skipping refresh test") + } + + srv := testSite(t) + defer srv.Close() + seed, _ := urlx.ParseSeed(srv.URL) + + out := t.TempDir() + cfg := DefaultConfig() + cfg.OutDir = out + cfg.Settle = 300 * time.Millisecond + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + if _, err := New(seed, cfg, t.Logf).Run(ctx); err != nil { + t.Fatalf("first run: %v", err) + } + + // Resume keeps everything in place but renders nothing new; refresh, on the + // same mirror, re-renders every page so changed content is pulled back in. + cfg.Refresh = true + res, err := New(seed, cfg, t.Logf).Run(ctx) + if err != nil { + t.Fatalf("refresh run: %v", err) + } + if res.Pages < 2 { + t.Fatalf("refresh should re-render every page, got %d", res.Pages) + } + if !fileExists(filepath.Join(out, seed.Hostname(), "index.html")) { + t.Error("refresh removed the mirror instead of overwriting in place") + } +} + func readFile(t *testing.T, path string) string { t.Helper() b, err := os.ReadFile(path) diff --git a/clone/config.go b/clone/config.go index 4bd419b..7eef526 100644 --- a/clone/config.go +++ b/clone/config.go @@ -4,12 +4,24 @@ package clone import ( + "os" "path/filepath" "time" "github.com/tamnd/kage/urlx" ) +// DefaultOutDir is where mirrors land unless --out overrides it: a per-user data +// directory ($HOME/data/kage) so clones from anywhere collect in one place, +// falling back to a local kage-out when the home directory cannot be resolved. +func DefaultOutDir() string { + home, err := os.UserHomeDir() + if err != nil || home == "" { + return "kage-out" + } + return filepath.Join(home, "data", "kage") +} + // Config is the full set of knobs for a clone run. DefaultConfig fills the // baseline; the CLI overlays flags on top. type Config struct { @@ -41,8 +53,16 @@ type Config struct { ChromeBin string ControlURL string - Resume bool - Force bool + // Resume loads the prior run's visited set and skips pages already written, + // so an interrupted or repeated clone picks up where it left off instead of + // refetching. Refresh forces every page to be re-rendered in place (the + // mirror is kept, files are overwritten) to pull in changed content. Force + // deletes the mirror first for a clean-slate clone. Persist writes the + // visited set back to state.json when the run ends. + Resume bool + Refresh bool + Force bool + Persist bool } // DefaultUserAgent is a current desktop Chrome UA, used by the asset fetcher and @@ -53,7 +73,7 @@ const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + // DefaultConfig returns the baseline configuration. func DefaultConfig() Config { return Config{ - OutDir: "kage-out", + OutDir: DefaultOutDir(), Reserved: urlx.DefaultReserved, Workers: 4, AssetWorkers: 8, @@ -68,6 +88,7 @@ func DefaultConfig() Config { FollowSitemap: true, Headless: true, Resume: true, + Persist: true, } } diff --git a/docs/content/reference/cli.md b/docs/content/reference/cli.md index 67d2353..8786739 100644 --- a/docs/content/reference/cli.md +++ b/docs/content/reference/cli.md @@ -24,9 +24,10 @@ images, and fonts, and writes a browsable mirror to `//`. | Flag | Default | Meaning | |------|---------|---------| -| `-o, --out` | `kage-out` | Output root; the mirror lands in `//` | +| `-o, --out` | `$HOME/data/kage` | Output root; the mirror lands in `//` | | `--reserved` | `_kage` | Reserved directory name for assets and crawl state | | `-f, --force` | `false` | Delete any existing mirror for the host before crawling | +| `--refresh` | `false` | Re-render every page in place to pull in changed content | | `--no-resume` | `false` | Do not read or write resume state | ### Scope diff --git a/docs/content/reference/configuration.md b/docs/content/reference/configuration.md index 9a2641a..da65a85 100644 --- a/docs/content/reference/configuration.md +++ b/docs/content/reference/configuration.md @@ -20,7 +20,8 @@ kage's launcher can download a private copy of Chromium on first use. ## Output layout -A clone of `example.com` lands under `kage-out/example.com/`: +A clone of `example.com` lands under `$HOME/data/kage/example.com/` (override the +root with `-o/--out`): ``` kage-out/example.com/ @@ -50,5 +51,19 @@ Key points: `style.css?v=3` is saved with a short hash suffix so two versions never collide. - **State lives in the mirror.** `_kage/state.json` records every page written, - which is what makes `--resume` able to skip completed work. Rename the reserved + which is what lets a repeated run skip completed work. Rename the reserved directory with `--reserved` if `_kage` would clash with a real path on the site. + +## Resume, refresh, and re-crawl + +A clone is idempotent: every page is keyed by the file it writes, so the same +page reached over `http` and `https`, with or without a trailing slash, or as +`/index.html` versus `/`, is fetched exactly once. Re-running picks the work back +up rather than starting over. + +| You want to… | Use | What happens | +|--------------|-----|--------------| +| Continue an interrupted crawl | *(default)* | Loads `state.json`, skips pages already written, fetches only what is missing | +| Pull in content that changed on the site | `--refresh` | Keeps the mirror, re-renders every page in place, overwrites with the new DOM | +| Start completely clean | `--force` | Deletes the host's mirror, then crawls from scratch | +| Run once and leave no trace | `--no-resume` | Skips nothing, writes no `state.json` | diff --git a/urlx/urlx.go b/urlx/urlx.go index a527984..5c41e44 100644 --- a/urlx/urlx.go +++ b/urlx/urlx.go @@ -206,7 +206,7 @@ func LocalPath(seedHost string, u *url.URL, kind Kind, reserved string) string { base = applyQuery(base, u) return joinClean(reserved, host, dir, base) default: // Page - dir := strings.Trim(u.Path, "/") + dir := collapseIndex(strings.Trim(u.Path, "/")) leaf := applyQuery("index.html", u) if strings.EqualFold(host, seedHost) { return joinClean(dir, leaf) @@ -217,6 +217,21 @@ func LocalPath(seedHost string, u *url.URL, kind Kind, reserved string) string { } } +// collapseIndex treats a directory-index document as the directory itself, so +// "/" and "/index.html" map to one page, and "/docs/index.html" collapses to +// "/docs". The argument is a path already trimmed of surrounding slashes. +func collapseIndex(p string) string { + for _, idx := range []string{"index.html", "index.htm"} { + if p == idx { + return "" + } + if rest, ok := strings.CutSuffix(p, "/"+idx); ok { + return rest + } + } + return p +} + // splitAsset breaks an asset URL path into a directory and a filename, inventing // a filename for directory-like or empty paths. func splitAsset(u *url.URL) (dir, base string) { diff --git a/urlx/urlx_test.go b/urlx/urlx_test.go index ed11f69..87b6353 100644 --- a/urlx/urlx_test.go +++ b/urlx/urlx_test.go @@ -139,6 +139,11 @@ func TestLocalPathPages(t *testing.T) { {"https://ex.com/docs/intro", "docs/intro/index.html"}, {"https://ex.com/a.html", "a.html/index.html"}, {"https://sub.ex.com/x", "sub.ex.com/x/index.html"}, + // A directory-index document is the directory itself, so it shares a + // path with the bare directory and with the http/https variant. + {"https://ex.com/index.html", "index.html"}, + {"http://ex.com/", "index.html"}, + {"https://ex.com/docs/index.html", "docs/index.html"}, } for _, c := range cases { got := LocalPath(seed, mustParse(t, c.u), Page, "")