package cli import ( "context" "errors" "fmt" "os" "time" "github.com/spf13/cobra" "github.com/tamnd/kage/clone" "github.com/tamnd/kage/urlx" ) // cloneFlags holds the parsed flag values for one invocation. type cloneFlags struct { out string reserved string workers int assetWorkers int browserPages int maxPages int maxDepth int traversal string maxAssetMB int64 timeout time.Duration settle time.Duration renderTO time.Duration scroll bool userAgent string subdomains bool scopePrefix string exclude []string noRobots bool noSitemap bool headful bool keepNoscript bool chromeBin string controlURL string noResume bool refresh bool force bool quiet bool } func newCloneCmd() *cobra.Command { f := &cloneFlags{} cmd := &cobra.Command{ Use: "clone ", Short: "Clone a site into a self-contained offline folder", Long: "clone fetches the seed URL, follows in-scope links, and writes a browsable\n" + "mirror to //. Every page is rendered in Chrome and stripped of\n" + "scripts; CSS, images, and fonts are downloaded and rewritten to local paths.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return runClone(cmd.Context(), args[0], f) }, } fs := cmd.Flags() fs.StringVarP(&f.out, "out", "o", clone.DefaultOutDir(), "output root; the mirror lands in //") fs.StringVar(&f.reserved, "reserved", urlx.DefaultReserved, "reserved dir for assets and state") fs.IntVar(&f.workers, "workers", 4, "concurrent page render workers") fs.IntVar(&f.assetWorkers, "asset-workers", 8, "concurrent asset download workers") fs.IntVar(&f.browserPages, "browser-pages", 4, "Chrome page-pool size") fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "stop after N pages (0 = unlimited)") fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)") fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order: bfs or dfs") fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB") fs.DurationVar(&f.timeout, "timeout", 30*time.Second, "per-request timeout") fs.DurationVar(&f.settle, "settle", 1500*time.Millisecond, "network-idle quiet period before snapshot") fs.DurationVar(&f.renderTO, "render-timeout", 30*time.Second, "hard cap per page render") fs.BoolVar(&f.scroll, "scroll", false, "auto-scroll each page to trigger lazy loading") fs.StringVar(&f.userAgent, "user-agent", clone.DefaultUserAgent, "User-Agent for asset and robots fetches") fs.BoolVar(&f.subdomains, "subdomains", false, "treat subdomains of the seed host as in scope") fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix") fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)") fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)") fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml") fs.BoolVar(&f.headful, "headful", false, "run Chrome with a visible window (debugging)") fs.BoolVar(&f.keepNoscript, "keep-noscript", false, "unwrap