Add the clone engine, CLI, tests, CI, and docs

kage renders every page in headless Chrome, snapshots the final DOM, strips all JavaScript, and localises CSS, images, and fonts so a site can be browsed offline as a plain folder of files. The engine is split into small packages: urlx deterministic URL to local-path mapping and scope rules sanitize remove scripts, on* handlers, and javascript: URLs asset rewrite HTML and CSS references, download assets browser headless Chrome pool over the DevTools protocol robots robots.txt matcher clone the orchestrator: a polite resumable breadth-first crawl The cli package wires a cobra and fang command surface with two commands, clone and serve. Every pure package has table tests; the browser and clone packages add Chrome-driven end-to-end tests that skip when no browser is present or under -short. CI runs gofmt, vet, build, race tests, golangci-lint, govulncheck, and a tidy check on Linux and macOS. A goreleaser config fans one tag out to archives, deb/rpm/apk, a Chromium-bundled GHCR image, and the package managers. A tago docs site builds to Pages and Cloudflare.
2026-06-14 18:22:25 +07:00
parent f7c5a9a1c6
commit e6afa91e09
48 changed files with 4697 additions and 0 deletions
@@ -0,0 +1,190 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tamnd/kage/clone"
+	"github.com/tamnd/kage/urlx"
+)
+
+// cloneFlags holds the parsed flag values for one invocation.
+type cloneFlags struct {
+	out          string
+	reserved     string
+	workers      int
+	assetWorkers int
+	browserPages int
+	maxPages     int
+	maxDepth     int
+	traversal    string
+	maxAssetMB   int64
+	timeout      time.Duration
+	settle       time.Duration
+	renderTO     time.Duration
+	scroll       bool
+	userAgent    string
+	subdomains   bool
+	scopePrefix  string
+	exclude      []string
+	noRobots     bool
+	noSitemap    bool
+	headful      bool
+	keepNoscript bool
+	chromeBin    string
+	controlURL   string
+	noResume     bool
+	force        bool
+	quiet        bool
+}
+
+func newCloneCmd() *cobra.Command {
+	f := &cloneFlags{}
+	cmd := &cobra.Command{
+		Use:   "clone <url>",
+		Short: "Clone a site into a self-contained offline folder",
+		Long: "clone fetches the seed URL, follows in-scope links, and writes a browsable\n" +
+			"mirror to <out>/<host>/. Every page is rendered in Chrome and stripped of\n" +
+			"scripts; CSS, images, and fonts are downloaded and rewritten to local paths.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runClone(cmd.Context(), args[0], f)
+		},
+	}
+	fs := cmd.Flags()
+	fs.StringVarP(&f.out, "out", "o", "kage-out", "output root; the mirror lands in <out>/<host>/")
+	fs.StringVar(&f.reserved, "reserved", urlx.DefaultReserved, "reserved dir for assets and state")
+	fs.IntVar(&f.workers, "workers", 4, "concurrent page render workers")
+	fs.IntVar(&f.assetWorkers, "asset-workers", 8, "concurrent asset download workers")
+	fs.IntVar(&f.browserPages, "browser-pages", 4, "Chrome page-pool size")
+	fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "stop after N pages (0 = unlimited)")
+	fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)")
+	fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order: bfs or dfs")
+	fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB")
+	fs.DurationVar(&f.timeout, "timeout", 30*time.Second, "per-request timeout")
+	fs.DurationVar(&f.settle, "settle", 1500*time.Millisecond, "network-idle quiet period before snapshot")
+	fs.DurationVar(&f.renderTO, "render-timeout", 30*time.Second, "hard cap per page render")
+	fs.BoolVar(&f.scroll, "scroll", false, "auto-scroll each page to trigger lazy loading")
+	fs.StringVar(&f.userAgent, "user-agent", clone.DefaultUserAgent, "User-Agent for asset and robots fetches")
+	fs.BoolVar(&f.subdomains, "subdomains", false, "treat subdomains of the seed host as in scope")
+	fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix")
+	fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)")
+	fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)")
+	fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml")
+	fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)")
+	fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap <noscript> content instead of dropping it")
+	fs.StringVar(&f.chromeBin, "chrome", "", "path to the Chrome/Chromium binary")
+	fs.StringVar(&f.controlURL, "control-url", "", "attach to an existing Chrome DevTools endpoint")
+	fs.BoolVar(&f.noResume, "no-resume", false, "do not reuse or write resume state")
+	fs.BoolVarP(&f.force, "force", "f", false, "delete any existing mirror for the host first")
+	fs.BoolVarP(&f.quiet, "quiet", "q", false, "suppress per-page progress lines")
+	return cmd
+}
+
+func runClone(ctx context.Context, arg string, f *cloneFlags) error {
+	seed, err := urlx.ParseSeed(arg)
+	if err != nil {
+		return fmt.Errorf("invalid url %q: %w", arg, err)
+	}
+
+	cfg := clone.DefaultConfig()
+	cfg.OutDir = f.out
+	cfg.Reserved = f.reserved
+	cfg.Workers = f.workers
+	cfg.AssetWorkers = f.assetWorkers
+	cfg.BrowserPages = f.browserPages
+	cfg.MaxPages = f.maxPages
+	cfg.MaxDepth = f.maxDepth
+	cfg.Traversal = f.traversal
+	cfg.MaxAssetBytes = f.maxAssetMB << 20
+	cfg.Timeout = f.timeout
+	cfg.Settle = f.settle
+	cfg.RenderTimeout = f.renderTO
+	cfg.Scroll = f.scroll
+	cfg.UserAgent = f.userAgent
+	cfg.IncludeSubdomains = f.subdomains
+	cfg.ScopePrefix = f.scopePrefix
+	cfg.ExcludePaths = f.exclude
+	cfg.RespectRobots = !f.noRobots
+	cfg.FollowSitemap = !f.noSitemap
+	cfg.Headless = !f.headful
+	cfg.KeepNoscript = f.keepNoscript
+	cfg.ChromeBin = f.chromeBin
+	cfg.ControlURL = f.controlURL
+	cfg.Resume = !f.noResume
+	cfg.Force = f.force
+
+	logf := func(format string, args ...any) {
+		if !f.quiet {
+			fmt.Fprintln(os.Stderr, styleDim.Render(fmt.Sprintf(format, args...)))
+		}
+	}
+
+	fmt.Fprintln(os.Stderr, styleTitle.Render("kage")+" cloning "+styleAccent.Render(seed.String()))
+
+	c := clone.New(seed, cfg, logf)
+
+	// Live progress ticker on a second line, refreshed every second.
+	tickCtx, stopTicker := context.WithCancel(ctx)
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		if f.quiet {
+			return
+		}
+		t := time.NewTicker(time.Second)
+		defer t.Stop()
+		for {
+			select {
+			case <-tickCtx.Done():
+				return
+			case <-t.C:
+				p := c.Snapshot()
+				fmt.Fprintf(os.Stderr, "\r%s", progressLine(p))
+			}
+		}
+	}()
+
+	res, runErr := c.Run(ctx)
+	stopTicker()
+	<-done
+	if !f.quiet {
+		fmt.Fprint(os.Stderr, "\r\033[K")
+	}
+
+	printSummary(res)
+
+	if runErr != nil && !errors.Is(runErr, context.Canceled) {
+		return runErr
+	}
+	if errors.Is(runErr, context.Canceled) {
+		fmt.Fprintln(os.Stderr, styleWarn.Render("interrupted; resume state saved (rerun to continue)"))
+	}
+	return nil
+}
+
+// progressLine renders the single-line live counter.
+func progressLine(p clone.Progress) string {
+	return styleDim.Render(fmt.Sprintf("pages %d  assets %d  errors %d  skipped %d",
+		p.Pages, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
+}
+
+// printSummary prints the final tally and where the mirror landed.
+func printSummary(res clone.Result) {
+	fmt.Fprintln(os.Stderr, styleOK.Render("done")+" "+styleTitle.Render(res.OutDir))
+	fmt.Fprintf(os.Stderr, "  %s %d   %s %d\n",
+		styleAccent.Render("pages"), res.Pages,
+		styleAccent.Render("assets"), res.Assets)
+	if res.PageErrors+res.AssetErrors > 0 {
+		fmt.Fprintf(os.Stderr, "  %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors)
+	}
+	if res.Skipped > 0 {
+		fmt.Fprintf(os.Stderr, "  %s %d\n", styleWarn.Render("skipped"), res.Skipped)
+	}
+	fmt.Fprintf(os.Stderr, "  open %s\n", styleAccent.Render("kage serve "+res.OutDir))
+}
@@ -0,0 +1,45 @@
+// Package cli wires kage's command surface: the cobra tree, the global flags,
+// and the fang-rendered help and errors. The actual work lives in the clone,
+// browser, sanitize, asset, and urlx packages; this layer only parses flags and
+// prints progress.
+package cli
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/charmbracelet/fang"
+	"github.com/spf13/cobra"
+)
+
+// Execute builds the root command and runs it through fang. main passes the
+// signal-aware context so Ctrl-C cancels the in-flight clone and flushes resume
+// state. It returns the process exit code.
+func Execute(ctx context.Context) int {
+	root := newRoot()
+	opts := []fang.Option{
+		fang.WithVersion(Version),
+	}
+	if err := fang.Execute(ctx, root, opts...); err != nil {
+		return 1
+	}
+	return 0
+}
+
+// newRoot assembles the command tree.
+func newRoot() *cobra.Command {
+	root := &cobra.Command{
+		Use:   "kage",
+		Short: "Clone any website for offline viewing, with the JavaScript stripped out",
+		Long: "kage (影, \"shadow\") renders each page in headless Chrome, snapshots the\n" +
+			"final DOM, removes every script and event handler, and localises the CSS,\n" +
+			"images, and fonts so the saved copy looks like the live site but runs no\n" +
+			"code. The result is a plain folder you can open straight from disk.",
+		Version:       fmt.Sprintf("%s (commit %s, built %s)", Version, Commit, Date),
+		SilenceUsage:  true,
+		SilenceErrors: true,
+	}
+	root.AddCommand(newCloneCmd())
+	root.AddCommand(newServeCmd())
+	return root
+}
@@ -0,0 +1,54 @@
+package cli
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+)
+
+func newServeCmd() *cobra.Command {
+	var addr string
+	cmd := &cobra.Command{
+		Use:   "serve [dir]",
+		Short: "Preview a cloned folder in your browser",
+		Long: "serve runs a local static file server over a cloned folder so you can click\n" +
+			"through the mirror exactly as a visitor would. With no dir it serves the\n" +
+			"current directory.",
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			dir := "."
+			if len(args) == 1 {
+				dir = args[0]
+			}
+			return runServe(dir, addr)
+		},
+	}
+	cmd.Flags().StringVarP(&addr, "addr", "a", "127.0.0.1:8800", "address to listen on")
+	return cmd
+}
+
+func runServe(dir, addr string) error {
+	info, err := os.Stat(dir)
+	if err != nil {
+		return fmt.Errorf("cannot serve %q: %w", dir, err)
+	}
+	if !info.IsDir() {
+		return fmt.Errorf("%q is not a directory", dir)
+	}
+	abs, _ := filepath.Abs(dir)
+
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		return fmt.Errorf("cannot listen on %s: %w", addr, err)
+	}
+
+	srv := &http.Server{Handler: http.FileServer(http.Dir(abs))}
+	fmt.Fprintln(os.Stderr, styleTitle.Render("kage serve")+" "+styleDim.Render(abs))
+	fmt.Fprintln(os.Stderr, "  open "+styleAccent.Render("http://"+ln.Addr().String()))
+	fmt.Fprintln(os.Stderr, styleDim.Render("  press Ctrl-C to stop"))
+	return srv.Serve(ln)
+}
@@ -0,0 +1,14 @@
+package cli
+
+import "charm.land/lipgloss/v2"
+
+// Styles for the human-readable progress and summary lines. They degrade to
+// plain text when the terminal has no colour profile.
+var (
+	styleTitle  = lipgloss.NewStyle().Bold(true)
+	styleAccent = lipgloss.NewStyle().Foreground(lipgloss.Color("12"))
+	styleOK     = lipgloss.NewStyle().Foreground(lipgloss.Color("10"))
+	styleWarn   = lipgloss.NewStyle().Foreground(lipgloss.Color("11"))
+	styleErr    = lipgloss.NewStyle().Foreground(lipgloss.Color("9"))
+	styleDim    = lipgloss.NewStyle().Foreground(lipgloss.Color("8"))
+)
@@ -0,0 +1,9 @@
+package cli
+
+// Build metadata, stamped via -ldflags at release time. goreleaser targets
+// github.com/tamnd/kage/cli.{Version,Commit,Date}.
+var (
+	Version = "dev"
+	Commit  = "none"
+	Date    = "unknown"
+)