Add the clone engine, CLI, tests, CI, and docs

kage renders every page in headless Chrome, snapshots the final
DOM, strips all JavaScript, and localises CSS, images, and fonts
so a site can be browsed offline as a plain folder of files.

The engine is split into small packages:

  urlx      deterministic URL to local-path mapping and scope rules
  sanitize  remove scripts, on* handlers, and javascript: URLs
  asset     rewrite HTML and CSS references, download assets
  browser   headless Chrome pool over the DevTools protocol
  robots    robots.txt matcher
  clone     the orchestrator: a polite resumable breadth-first crawl

The cli package wires a cobra and fang command surface with two
commands, clone and serve. Every pure package has table tests; the
browser and clone packages add Chrome-driven end-to-end tests that
skip when no browser is present or under -short.

CI runs gofmt, vet, build, race tests, golangci-lint, govulncheck,
and a tidy check on Linux and macOS. A goreleaser config fans one
tag out to archives, deb/rpm/apk, a Chromium-bundled GHCR image,
and the package managers. A tago docs site builds to Pages and
Cloudflare.

clone/cloner.go

@@ -0,0 +1,377 @@
+package clone
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/tamnd/kage/asset"
+	"github.com/tamnd/kage/browser"
+	"github.com/tamnd/kage/robots"
+	"github.com/tamnd/kage/sanitize"
+	"github.com/tamnd/kage/urlx"
+	"golang.org/x/net/html"
+)
+
+// Logf is an optional sink for human-readable progress lines.
+type Logf func(format string, args ...any)
+
+// Cloner runs one clone. Build it with New, then call Run.
+type Cloner struct {
+	cfg      Config
+	seed     *url.URL
+	seedHost string
+	outRoot  string
+	statePth string
+
+	pool   *browser.Pool
+	dl     *asset.Downloader
+	httpC  *http.Client
+	robots *robots.Matcher
+	front  *frontier
+	stats  stats
+	logf   Logf
+
+	mu         sync.Mutex
+	seenAssets map[string]bool
+	enqueued   int // pages offered to the queue
+	wg         sync.WaitGroup
+	pageJobs   chan pageItem
+	assetJobs  chan assetItem
+}
+
+type pageItem struct {
+	u     *url.URL
+	depth int
+}
+
+type assetItem struct {
+	u       *url.URL
+	referer string
+}
+
+// New builds a Cloner for seed under cfg. It does not touch the network until
+// Run is called.
+func New(seed *url.URL, cfg Config, logf Logf) *Cloner {
+	if logf == nil {
+		logf = func(string, ...any) {}
+	}
+	host := seed.Hostname()
+	outRoot := cfg.HostDir(host)
+	return &Cloner{
+		cfg:        cfg,
+		seed:       seed,
+		seedHost:   host,
+		outRoot:    outRoot,
+		statePth:   filepath.Join(outRoot, cfg.Reserved, "state.json"),
+		dl:         asset.NewDownloader(cfg.UserAgent, cfg.Timeout, cfg.MaxAssetBytes),
+		httpC:      &http.Client{Timeout: cfg.Timeout},
+		robots:     robots.AllowAll(),
+		front:      newFrontier(),
+		logf:       logf,
+		seenAssets: map[string]bool{},
+		pageJobs:   make(chan pageItem),
+		assetJobs:  make(chan assetItem),
+	}
+}
+
+// Snapshot returns the current progress, for a CLI ticker.
+func (c *Cloner) Snapshot() Progress { return c.stats.snapshot() }
+
+// Run executes the clone until the frontier drains, MaxPages is hit, or ctx is
+// cancelled (which flushes the resume state). It returns the final Result.
+func (c *Cloner) Run(ctx context.Context) (Result, error) {
+	if c.cfg.Force {
+		_ = os.RemoveAll(c.outRoot)
+	}
+	if c.cfg.Resume {
+		if err := c.front.load(c.statePth); err != nil {
+			c.logf("resume: could not load state: %v", err)
+		} else if n := c.front.visitedCount(); n > 0 {
+			c.logf("resume: %d pages already done", n)
+		}
+	}
+
+	c.pool = browser.New(browser.Options{
+		Headless:      c.cfg.Headless,
+		Workers:       c.cfg.BrowserPages,
+		Settle:        c.cfg.Settle,
+		RenderTimeout: c.cfg.RenderTimeout,
+		Scroll:        c.cfg.Scroll,
+		ChromeBin:     c.cfg.ChromeBin,
+		ControlURL:    c.cfg.ControlURL,
+	})
+	defer func() { _ = c.pool.Close() }()
+
+	c.loadRobots(ctx)
+
+	// Start workers.
+	var workers sync.WaitGroup
+	for range max1(c.cfg.Workers) {
+		workers.Go(func() {
+			for j := range c.pageJobs {
+				c.processPage(ctx, j)
+				c.wg.Done()
+			}
+		})
+	}
+	for range max1(c.cfg.AssetWorkers) {
+		workers.Go(func() {
+			for j := range c.assetJobs {
+				c.processAsset(ctx, j)
+				c.wg.Done()
+			}
+		})
+	}
+
+	// Seed.
+	c.enqueuePage(ctx, c.seed, 0)
+	if c.cfg.FollowSitemap {
+		c.seedSitemaps(ctx)
+	}
+
+	// Close the job channels once every outstanding item is processed.
+	go func() {
+		c.wg.Wait()
+		close(c.pageJobs)
+		close(c.assetJobs)
+	}()
+	workers.Wait()
+
+	if err := c.front.save(c.statePth); err != nil {
+		c.logf("could not save resume state: %v", err)
+	}
+
+	res := Result{Progress: c.stats.snapshot(), OutDir: c.outRoot}
+	if ctx.Err() != nil {
+		return res, ctx.Err()
+	}
+	return res, nil
+}
+
+// loadRobots fetches and parses robots.txt (unless disabled) and seeds the
+// sitemap list it advertises.
+func (c *Cloner) loadRobots(ctx context.Context) {
+	if !c.cfg.RespectRobots {
+		return
+	}
+	robotsURL := c.seed.Scheme + "://" + c.seed.Host + "/robots.txt"
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, robotsURL, nil)
+	if err != nil {
+		return
+	}
+	req.Header.Set("User-Agent", c.cfg.UserAgent)
+	resp, err := c.httpC.Do(req)
+	if err != nil {
+		return
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		return
+	}
+	data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if err != nil {
+		return
+	}
+	c.robots = robots.Parse(string(data), "kage")
+}
+
+// seedSitemaps adds in-scope sitemap URLs (from robots and the default path) to
+// the frontier.
+func (c *Cloner) seedSitemaps(ctx context.Context) {
+	seeds := append([]string{}, c.robots.Sitemaps...)
+	seeds = append(seeds, c.seed.Scheme+"://"+c.seed.Host+"/sitemap.xml")
+	locs := collectSitemaps(ctx, c.httpC, c.cfg.UserAgent, seeds)
+	added := 0
+	for _, loc := range locs {
+		u, err := urlx.Normalize(c.seed, loc)
+		if err != nil {
+			continue
+		}
+		if urlx.InScope(c.seed, u, c.cfg.scope()) && urlx.LikelyPage(u) {
+			if c.enqueuePage(ctx, u, 1) {
+				added++
+			}
+		}
+	}
+	if added > 0 {
+		c.logf("sitemap: seeded %d URLs", added)
+	}
+}
+
+// processPage renders one page, rewrites its references to local paths, strips
+// every script, and writes the result.
+func (c *Cloner) processPage(ctx context.Context, j pageItem) {
+	if ctx.Err() != nil {
+		return
+	}
+	key := urlx.Key(j.u)
+	if c.cfg.RespectRobots && !c.robots.Allowed(j.u.Path) {
+		c.stats.skipped.Add(1)
+		return
+	}
+
+	res, err := c.pool.Render(ctx, j.u.String())
+	if err != nil {
+		c.stats.pageErrors.Add(1)
+		c.logf("page error %s: %v", j.u, err)
+		return
+	}
+
+	root, err := html.Parse(strings.NewReader(res.HTML))
+	if err != nil {
+		c.stats.pageErrors.Add(1)
+		c.logf("parse error %s: %v", j.u, err)
+		return
+	}
+
+	localFile := urlx.LocalPath(c.seedHost, j.u, urlx.Page, c.cfg.Reserved)
+	fileDir := urlx.Dir(localFile)
+
+	sink := func(u *url.URL, kind urlx.Kind) string {
+		switch kind {
+		case urlx.Page:
+			if urlx.InScope(c.seed, u, c.cfg.scope()) {
+				c.enqueuePage(ctx, u, j.depth+1)
+				local := urlx.LocalPath(c.seedHost, u, urlx.Page, c.cfg.Reserved)
+				return urlx.Rel(fileDir, local)
+			}
+			return u.String() // external page link stays on the live web
+		default: // Asset
+			c.enqueueAsset(ctx, u, j.u.String())
+			local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
+			return urlx.Rel(fileDir, local)
+		}
+	}
+
+	asset.RewriteHTML(root, j.u, sink)
+	sanitize.CleanTree(root, sanitize.Options{
+		KeepNoscript: c.cfg.KeepNoscript,
+		Banner:       "cloned by kage from " + j.u.String(),
+	})
+
+	var buf strings.Builder
+	if err := html.Render(&buf, root); err != nil {
+		c.stats.pageErrors.Add(1)
+		return
+	}
+	if err := c.writeFile(localFile, []byte(buf.String())); err != nil {
+		c.stats.pageErrors.Add(1)
+		c.logf("write error %s: %v", localFile, err)
+		return
+	}
+	c.front.markVisited(key)
+	c.stats.pages.Add(1)
+}
+
+// processAsset downloads one asset, rewriting CSS references on the way, and
+// writes it to its deterministic local path.
+func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
+	if ctx.Err() != nil {
+		return
+	}
+	res, err := c.dl.Get(ctx, j.u, j.referer)
+	if err != nil {
+		c.stats.assetErrors.Add(1)
+		c.logf("asset error %s: %v", j.u, err)
+		return
+	}
+
+	localFile := urlx.LocalPath(c.seedHost, j.u, urlx.Asset, c.cfg.Reserved)
+	body := res.Body
+	if res.IsCSS {
+		fileDir := urlx.Dir(localFile)
+		cssSink := func(u *url.URL, _ urlx.Kind) string {
+			c.enqueueAsset(ctx, u, j.u.String())
+			local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
+			return urlx.Rel(fileDir, local)
+		}
+		body = asset.RewriteCSS(body, j.u, cssSink)
+	}
+	if err := c.writeFile(localFile, body); err != nil {
+		c.stats.assetErrors.Add(1)
+		c.logf("write error %s: %v", localFile, err)
+		return
+	}
+	c.stats.assets.Add(1)
+}
+
+// enqueuePage offers a page URL to the frontier, honouring the visited set, the
+// depth cap, and the page budget. It reports whether the page was newly queued.
+func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool {
+	if c.cfg.MaxDepth > 0 && depth > c.cfg.MaxDepth {
+		return false
+	}
+	key := urlx.Key(u)
+	if c.front.isVisited(key) {
+		return false
+	}
+	if !c.front.offer(key) {
+		return false
+	}
+	c.mu.Lock()
+	if c.cfg.MaxPages > 0 && c.enqueued >= c.cfg.MaxPages {
+		c.mu.Unlock()
+		return false
+	}
+	c.enqueued++
+	c.mu.Unlock()
+
+	c.wg.Add(1)
+	go func() {
+		select {
+		case c.pageJobs <- pageItem{u: u, depth: depth}:
+		case <-ctx.Done():
+			c.wg.Done()
+		}
+	}()
+	return true
+}
+
+// enqueueAsset offers an asset URL for download, deduping by canonical URL.
+func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) {
+	key := urlx.Key(u)
+	c.mu.Lock()
+	if c.seenAssets[key] {
+		c.mu.Unlock()
+		return
+	}
+	c.seenAssets[key] = true
+	c.mu.Unlock()
+
+	c.wg.Add(1)
+	go func() {
+		select {
+		case c.assetJobs <- assetItem{u: u, referer: referer}:
+		case <-ctx.Done():
+			c.wg.Done()
+		}
+	}()
+}
+
+// writeFile writes data to a slash path relative to the mirror root, creating
+// parent directories. The path is cleaned so it can never escape the root.
+func (c *Cloner) writeFile(relSlash string, data []byte) error {
+	rel := filepath.FromSlash(relSlash)
+	full := filepath.Join(c.outRoot, rel)
+	if !strings.HasPrefix(full, filepath.Clean(c.outRoot)+string(os.PathSeparator)) && full != c.outRoot {
+		return fmt.Errorf("refusing to write outside mirror root: %s", relSlash)
+	}
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		return err
+	}
+	return os.WriteFile(full, data, 0o644)
+}
+
+func max1(n int) int {
+	if n < 1 {
+		return 1
+	}
+	return n
+}

clone/cloner_test.go

@@ -0,0 +1,226 @@
+package clone
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/tamnd/kage/browser"
+	"github.com/tamnd/kage/urlx"
+)
+
+// testSite is a tiny two-page site with a stylesheet, an image, an inline
+// script, an onclick handler, and a javascript: link, so a full clone exercises
+// rendering, asset localisation, and JavaScript stripping at once.
+func testSite(t *testing.T) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write([]byte(`<!doctype html><html><head>
+<link rel="stylesheet" href="/site.css">
+<script src="/app.js"></script>
+</head><body>
+<h1>Home</h1>
+<img src="/logo.png" alt="logo">
+<a href="/about">About</a>
+<a href="javascript:void(0)" onclick="boom()">Danger</a>
+<script>console.log("inline")</script>
+</body></html>`))
+	})
+	mux.HandleFunc("/about", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write([]byte(`<!doctype html><html><head>
+<link rel="stylesheet" href="/site.css">
+</head><body><h1>About</h1><a href="/">Home</a></body></html>`))
+	})
+	mux.HandleFunc("/site.css", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/css")
+		_, _ = w.Write([]byte(`body{background:url("/bg.png")} h1{color:red}`))
+	})
+	mux.HandleFunc("/app.js", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/javascript")
+		_, _ = w.Write([]byte(`document.body.dataset.ran = "1";`))
+	})
+	mux.HandleFunc("/logo.png", servePNG)
+	mux.HandleFunc("/bg.png", servePNG)
+	mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("User-agent: *\nAllow: /\n"))
+	})
+	return httptest.NewServer(mux)
+}
+
+// a 1x1 transparent PNG.
+var pngBytes = []byte{
+	0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
+	0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
+	0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89, 0x00, 0x00, 0x00,
+	0x0a, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0x00, 0x01, 0x00, 0x00,
+	0x05, 0x00, 0x01, 0x0d, 0x0a, 0x2d, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x49,
+	0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
+}
+
+func servePNG(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "image/png")
+	_, _ = w.Write(pngBytes)
+}
+
+func TestCloneEndToEnd(t *testing.T) {
+	if testing.Short() {
+		t.Skip("clone end-to-end drives Chrome; skipped under -short")
+	}
+	if _, ok := browser.LookChrome(); !ok {
+		t.Skip("no Chrome/Chromium found; skipping clone end-to-end")
+	}
+
+	srv := testSite(t)
+	defer srv.Close()
+
+	seed, err := urlx.ParseSeed(srv.URL)
+	if err != nil {
+		t.Fatalf("parse seed: %v", err)
+	}
+
+	out := t.TempDir()
+	cfg := DefaultConfig()
+	cfg.OutDir = out
+	cfg.Settle = 300 * time.Millisecond
+	cfg.RenderTimeout = 20 * time.Second
+	cfg.Timeout = 10 * time.Second
+
+	c := New(seed, cfg, t.Logf)
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+
+	res, err := c.Run(ctx)
+	if err != nil {
+		t.Fatalf("run: %v", err)
+	}
+	if res.Pages < 2 {
+		t.Fatalf("expected at least 2 pages, got %d", res.Pages)
+	}
+
+	root := res.OutDir
+	indexPath := filepath.Join(root, "index.html")
+	body := readFile(t, indexPath)
+
+	// JavaScript is gone: no <script>, no onclick, no javascript: URL.
+	if strings.Contains(strings.ToLower(body), "<script") {
+		t.Error("index.html still contains a <script> tag")
+	}
+	if strings.Contains(strings.ToLower(body), "onclick") {
+		t.Error("index.html still contains an onclick handler")
+	}
+	if strings.Contains(strings.ToLower(body), "javascript:") {
+		t.Error("index.html still contains a javascript: URL")
+	}
+
+	// Layout is preserved: the stylesheet link survives and points local.
+	if !strings.Contains(body, "stylesheet") {
+		t.Error("index.html lost its stylesheet link")
+	}
+	if strings.Contains(body, srv.URL+"/site.css") {
+		t.Error("stylesheet still points at the live origin")
+	}
+
+	// The about page and the localised assets exist on disk.
+	if !fileExists(filepath.Join(root, "about", "index.html")) {
+		t.Error("about page was not written")
+	}
+	assetDir := filepath.Join(root, cfg.Reserved)
+	if !anyFileUnder(t, assetDir, "site.css") {
+		t.Error("site.css was not downloaded")
+	}
+	if !anyFileUnder(t, assetDir, "logo.png") {
+		t.Error("logo.png was not downloaded")
+	}
+
+	// The localised CSS had its url() rewritten away from the origin.
+	css := readAnyFile(t, assetDir, "site.css")
+	if strings.Contains(css, srv.URL) {
+		t.Error("site.css still references the live origin in url()")
+	}
+}
+
+func TestCloneResumeSkipsVisited(t *testing.T) {
+	if testing.Short() {
+		t.Skip("resume test drives Chrome; skipped under -short")
+	}
+	if _, ok := browser.LookChrome(); !ok {
+		t.Skip("no Chrome/Chromium found; skipping resume test")
+	}
+
+	srv := testSite(t)
+	defer srv.Close()
+	seed, _ := urlx.ParseSeed(srv.URL)
+
+	out := t.TempDir()
+	cfg := DefaultConfig()
+	cfg.OutDir = out
+	cfg.Settle = 300 * time.Millisecond
+
+	c1 := New(seed, cfg, t.Logf)
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+	if _, err := c1.Run(ctx); err != nil {
+		t.Fatalf("first run: %v", err)
+	}
+
+	// Second run with resume on should find the state and re-render nothing new.
+	c2 := New(seed, cfg, t.Logf)
+	res2, err := c2.Run(ctx)
+	if err != nil {
+		t.Fatalf("second run: %v", err)
+	}
+	if res2.Pages != 0 {
+		t.Fatalf("resume should skip all visited pages, but rendered %d", res2.Pages)
+	}
+}
+
+func readFile(t *testing.T, path string) string {
+	t.Helper()
+	b, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	return string(b)
+}
+
+func fileExists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+func anyFileUnder(t *testing.T, dir, name string) bool {
+	t.Helper()
+	found := false
+	_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
+		if err == nil && !d.IsDir() && strings.HasSuffix(p, name) {
+			found = true
+		}
+		return nil
+	})
+	return found
+}
+
+func readAnyFile(t *testing.T, dir, name string) string {
+	t.Helper()
+	var out string
+	_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
+		if err == nil && !d.IsDir() && strings.HasSuffix(p, name) {
+			b, _ := os.ReadFile(p)
+			out = string(b)
+		}
+		return nil
+	})
+	return out
+}

clone/config.go

@@ -0,0 +1,86 @@
+// Package clone is kage's engine: it ties the Chrome pool, the JavaScript
+// stripper, the asset localiser, and the URL↔path mapper into one resumable,
+// polite crawl that turns a live site into a browsable offline folder.
+package clone
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/tamnd/kage/urlx"
+)
+
+// Config is the full set of knobs for a clone run. DefaultConfig fills the
+// baseline; the CLI overlays flags on top.
+type Config struct {
+	OutDir   string // output root; the mirror lands in <OutDir>/<host>/
+	Reserved string // reserved dir name for assets and state (default "_kage")
+
+	Workers       int // page render workers
+	AssetWorkers  int // HTTP asset download workers
+	BrowserPages  int // Chrome page-pool size
+	MaxPages      int // stop after N pages (0 = unlimited)
+	MaxDepth      int // BFS/DFS depth cap (0 = unlimited)
+	Traversal     string
+	MaxAssetBytes int64
+
+	Timeout       time.Duration // per HTTP request
+	Settle        time.Duration // network-idle quiet period
+	RenderTimeout time.Duration // hard cap per page render
+	Scroll        bool
+
+	UserAgent         string
+	IncludeSubdomains bool
+	ScopePrefix       string
+	ExcludePaths      []string
+
+	RespectRobots bool
+	FollowSitemap bool
+	Headless      bool
+	KeepNoscript  bool
+	ChromeBin     string
+	ControlURL    string
+
+	Resume bool
+	Force  bool
+}
+
+// DefaultUserAgent is a current desktop Chrome UA, used by the asset fetcher and
+// the robots fetch so a site treats kage like the browser it drives.
+const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +
+	"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+
+// DefaultConfig returns the baseline configuration.
+func DefaultConfig() Config {
+	return Config{
+		OutDir:        "kage-out",
+		Reserved:      urlx.DefaultReserved,
+		Workers:       4,
+		AssetWorkers:  8,
+		BrowserPages:  4,
+		MaxAssetBytes: 25 << 20,
+		Traversal:     "bfs",
+		Timeout:       30 * time.Second,
+		Settle:        1500 * time.Millisecond,
+		RenderTimeout: 30 * time.Second,
+		UserAgent:     DefaultUserAgent,
+		RespectRobots: true,
+		FollowSitemap: true,
+		Headless:      true,
+		Resume:        true,
+	}
+}
+
+// HostDir returns the mirror root for a seed host: <OutDir>/<host>.
+func (c Config) HostDir(host string) string {
+	return filepath.Join(c.OutDir, host)
+}
+
+// scope builds the urlx scope config from the run config.
+func (c Config) scope() urlx.ScopeConfig {
+	return urlx.ScopeConfig{
+		IncludeSubdomains: c.IncludeSubdomains,
+		ScopePrefix:       c.ScopePrefix,
+		ExcludePaths:      c.ExcludePaths,
+	}
+}

clone/frontier.go

@@ -0,0 +1,106 @@
+package clone
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"sort"
+	"sync"
+)
+
+// frontier is the deduped set of page URLs kage has already seen. It is small,
+// concurrency-safe, and persists to disk so --resume can skip work already done.
+// The actual queueing is handled by the cloner's channels; the frontier only
+// answers "is this URL new?" and remembers the answer.
+type frontier struct {
+	mu      sync.Mutex
+	seen    map[string]bool // queued or visited
+	visited map[string]bool // fully written
+}
+
+func newFrontier() *frontier {
+	return &frontier{seen: map[string]bool{}, visited: map[string]bool{}}
+}
+
+// offer reports whether key is new (and records it as seen). A repeated key
+// returns false so it is enqueued only once.
+func (f *frontier) offer(key string) bool {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.seen[key] {
+		return false
+	}
+	f.seen[key] = true
+	return true
+}
+
+// markVisited records that a page was written.
+func (f *frontier) markVisited(key string) {
+	f.mu.Lock()
+	f.visited[key] = true
+	f.mu.Unlock()
+}
+
+// isVisited reports whether a page was already written in a previous run.
+func (f *frontier) isVisited(key string) bool {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.visited[key]
+}
+
+func (f *frontier) visitedCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.visited)
+}
+
+// state is the JSON shape persisted for resume.
+type state struct {
+	Visited []string `json:"visited"`
+}
+
+// load reads a previously saved visited set; a missing file is not an error.
+func (f *frontier) load(path string) error {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	var s state
+	if err := json.Unmarshal(data, &s); err != nil {
+		return err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	for _, v := range s.Visited {
+		f.visited[v] = true
+		f.seen[v] = true
+	}
+	return nil
+}
+
+// save writes the visited set atomically (write temp, rename).
+func (f *frontier) save(path string) error {
+	f.mu.Lock()
+	visited := make([]string, 0, len(f.visited))
+	for v := range f.visited {
+		visited = append(visited, v)
+	}
+	f.mu.Unlock()
+	sort.Strings(visited)
+
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return err
+	}
+	data, err := json.MarshalIndent(state{Visited: visited}, "", "  ")
+	if err != nil {
+		return err
+	}
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, data, 0o644); err != nil {
+		return err
+	}
+	return os.Rename(tmp, path)
+}

clone/frontier_test.go

@@ -0,0 +1,67 @@
+package clone
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+func TestFrontierOfferDedups(t *testing.T) {
+	f := newFrontier()
+	if !f.offer("a") {
+		t.Fatal("first offer of a should be new")
+	}
+	if f.offer("a") {
+		t.Fatal("second offer of a should be a duplicate")
+	}
+	if !f.offer("b") {
+		t.Fatal("first offer of b should be new")
+	}
+}
+
+func TestFrontierVisited(t *testing.T) {
+	f := newFrontier()
+	if f.isVisited("x") {
+		t.Fatal("x should not be visited yet")
+	}
+	f.markVisited("x")
+	if !f.isVisited("x") {
+		t.Fatal("x should be visited after markVisited")
+	}
+	if f.visitedCount() != 1 {
+		t.Fatalf("visitedCount = %d, want 1", f.visitedCount())
+	}
+}
+
+func TestFrontierSaveLoadRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "sub", "state.json")
+
+	f := newFrontier()
+	f.markVisited("https://example.com/")
+	f.markVisited("https://example.com/about")
+	if err := f.save(path); err != nil {
+		t.Fatalf("save: %v", err)
+	}
+
+	g := newFrontier()
+	if err := g.load(path); err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	if g.visitedCount() != 2 {
+		t.Fatalf("loaded visitedCount = %d, want 2", g.visitedCount())
+	}
+	if !g.isVisited("https://example.com/about") {
+		t.Fatal("about should be visited after load")
+	}
+	// A loaded visited URL is also seen, so it is not re-offered.
+	if g.offer("https://example.com/about") {
+		t.Fatal("a loaded URL should not be offered again")
+	}
+}
+
+func TestFrontierLoadMissingIsNotError(t *testing.T) {
+	f := newFrontier()
+	if err := f.load(filepath.Join(t.TempDir(), "nope.json")); err != nil {
+		t.Fatalf("loading a missing file should be a no-op, got %v", err)
+	}
+}

clone/sitemap.go

@@ -0,0 +1,83 @@
+package clone
+
+import (
+	"context"
+	"encoding/xml"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// sitemapDoc covers both a urlset and a sitemapindex: both carry <loc> elements,
+// so one shape extracts the URLs from either.
+type sitemapDoc struct {
+	URLs     []sitemapEntry `xml:"url"`
+	Sitemaps []sitemapEntry `xml:"sitemap"`
+}
+
+type sitemapEntry struct {
+	Loc string `xml:"loc"`
+}
+
+// fetchSitemap downloads and parses one sitemap, returning page locations and,
+// for a sitemap index, the child sitemap URLs.
+func fetchSitemap(ctx context.Context, client *http.Client, ua, sitemapURL string) (locs, children []string) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, sitemapURL, nil)
+	if err != nil {
+		return nil, nil
+	}
+	if ua != "" {
+		req.Header.Set("User-Agent", ua)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, nil
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		return nil, nil
+	}
+	var doc sitemapDoc
+	dec := xml.NewDecoder(resp.Body)
+	if err := dec.Decode(&doc); err != nil {
+		return nil, nil
+	}
+	for _, u := range doc.URLs {
+		if s := strings.TrimSpace(u.Loc); s != "" {
+			locs = append(locs, s)
+		}
+	}
+	for _, sm := range doc.Sitemaps {
+		if s := strings.TrimSpace(sm.Loc); s != "" {
+			children = append(children, s)
+		}
+	}
+	return locs, children
+}
+
+// collectSitemaps walks a set of seed sitemap URLs (following index files one
+// level deep) and returns all discovered page locations, bounded by a deadline.
+func collectSitemaps(ctx context.Context, client *http.Client, ua string, seeds []string) []string {
+	ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
+	defer cancel()
+
+	var out []string
+	seen := map[string]bool{}
+	queue := append([]string{}, seeds...)
+	for len(queue) > 0 && ctx.Err() == nil {
+		sm := queue[0]
+		queue = queue[1:]
+		if seen[sm] {
+			continue
+		}
+		seen[sm] = true
+		locs, children := fetchSitemap(ctx, client, ua, sm)
+		out = append(out, locs...)
+		for _, c := range children {
+			if !seen[c] {
+				queue = append(queue, c)
+			}
+		}
+	}
+	return out
+}

clone/stats.go

@@ -0,0 +1,37 @@
+package clone
+
+import "sync/atomic"
+
+// stats are the live counters of a run, read by the CLI's progress ticker.
+type stats struct {
+	pages       atomic.Int64
+	assets      atomic.Int64
+	pageErrors  atomic.Int64
+	assetErrors atomic.Int64
+	skipped     atomic.Int64 // robots-disallowed or out of budget
+}
+
+// Progress is a snapshot of a run for display.
+type Progress struct {
+	Pages       int64
+	Assets      int64
+	PageErrors  int64
+	AssetErrors int64
+	Skipped     int64
+}
+
+func (s *stats) snapshot() Progress {
+	return Progress{
+		Pages:       s.pages.Load(),
+		Assets:      s.assets.Load(),
+		PageErrors:  s.pageErrors.Load(),
+		AssetErrors: s.assetErrors.Load(),
+		Skipped:     s.skipped.Load(),
+	}
+}
+
+// Result is the final outcome returned by Run.
+type Result struct {
+	Progress
+	OutDir string
+}