package cli import ( "context" "errors" "fmt" "os" "strings" "time" "github.com/spf13/cobra" "github.com/tamnd/kage/clone" "github.com/tamnd/kage/urlx" ) // cloneFlags holds the parsed flag values for one invocation. type cloneFlags struct { out string reserved string workers int assetWorkers int browserPages int maxPages int maxDepth int traversal string maxAssetMB int64 keepMedia bool skipExt []string allAssetHosts bool timeout time.Duration settle time.Duration renderTO time.Duration scroll bool userAgent string subdomains bool scopePrefix string exclude []string noRobots bool crawlDelay time.Duration noSitemap bool headful bool keepNoscript bool mobileReadable bool chromeBin string controlURL string noResume bool refresh bool force bool quiet bool } func newCloneCmd() *cobra.Command { f := &cloneFlags{} cmd := &cobra.Command{ Use: "clone ", Short: "Clone a site into a self-contained offline folder", Long: "clone fetches the seed URL, follows in-scope links, and writes a browsable\n" + "mirror to //. Every page is rendered in Chrome and stripped of\n" + "scripts; CSS, images, and fonts are downloaded and rewritten to local paths.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return runClone(cmd.Context(), args[0], f) }, } fs := cmd.Flags() fs.StringVarP(&f.out, "out", "o", clone.DefaultOutDir(), "output root; the mirror lands in //") fs.StringVar(&f.reserved, "reserved", urlx.DefaultReserved, "reserved dir for assets and state") fs.IntVar(&f.workers, "workers", 4, "concurrent page render workers") fs.IntVar(&f.assetWorkers, "asset-workers", 8, "concurrent asset download workers") fs.IntVar(&f.browserPages, "browser-pages", 4, "Chrome page-pool size") fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "stop after N pages (0 = unlimited)") fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)") fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order: bfs or dfs") fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB (left on the live web)") fs.BoolVar(&f.keepMedia, "keep-media", false, "download bulk media, installers, and PDFs instead of leaving them remote") fs.StringSliceVar(&f.skipExt, "skip-ext", nil, "extra asset extensions to leave remote, e.g. .svg (repeatable)") fs.BoolVar(&f.allAssetHosts, "all-asset-hosts", false, "localize assets from any host, not just the seed's domain") fs.DurationVar(&f.timeout, "timeout", 30*time.Second, "per-request timeout") fs.DurationVar(&f.settle, "settle", 1500*time.Millisecond, "network-idle quiet period before snapshot") fs.DurationVar(&f.renderTO, "render-timeout", 30*time.Second, "hard cap per page render") fs.BoolVar(&f.scroll, "scroll", false, "auto-scroll each page to trigger lazy loading") fs.StringVar(&f.userAgent, "user-agent", clone.DefaultUserAgent, "User-Agent for asset and robots fetches") fs.BoolVar(&f.subdomains, "subdomains", false, "treat subdomains of the seed host as in scope") fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix") fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)") fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)") fs.DurationVar(&f.crawlDelay, "crawl-delay", 0, "override robots.txt Crawl-delay between page starts (0 = use robots.txt)") fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml") fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)") fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap