frostbyte_neo/kage
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Count real pages and store identical pages once (#20)

Browse Source 
Two related changes for crawling faceted sites, where one path spawns
thousands of ?q=.../?page=... URLs that all render the same page.

The progress line was counting every written file, so the page number ran
far ahead of the site's real size. It now shows distinct URL paths as
"pages" and folds the query-string permutations into a separate "variants"
count, so the live counter tracks real pages and is easy to read.

Pages with identical bytes are now stored once. The first page with a given
content is written normally; a later page with the same bytes becomes a hard
link to it, so duplicate content never takes disk twice. Links still resolve
because each variant keeps its own path. When hard links are unsupported the
bytes are written, so correctness never depends on the link. The summary
reports how many pages were deduped.
This commit is contained in:
Tam Nguyen Duc
2026-06-15 19:07:49 +07:00
committed by GitHub
parent 8833a2b8f8
commit 7dc3621ced
5 changed files with 211 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/clone.go
+15 -3
View File
@@ -172,18 +172,30 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
return nil
}
// progressLine renders the single-line live counter.
// progressLine renders the single-line live counter. "pages" is the count of
// real pages (distinct paths); when a faceted site spawns query-string variants
// they are shown separately so the page number stays easy to read.
func progressLine(p clone.Progress) string {
if variants := p.Pages - p.PagePaths; variants > 0 {
return styleDim.Render(fmt.Sprintf("pages %d variants %d assets %d errors %d skipped %d",
p.PagePaths, variants, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
}
return styleDim.Render(fmt.Sprintf("pages %d assets %d errors %d skipped %d",
p.Pages, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
p.PagePaths, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
}
// printSummary prints the final tally and where the mirror landed.
func printSummary(res clone.Result) {
fmt.Fprintln(os.Stderr, styleOK.Render("done")+" "+styleTitle.Render(res.OutDir))
fmt.Fprintf(os.Stderr, " %s %d %s %d\n",
styleAccent.Render("pages"), res.Pages,
styleAccent.Render("pages"), res.PagePaths,
styleAccent.Render("assets"), res.Assets)
if variants := res.Pages - res.PagePaths; variants > 0 {
fmt.Fprintf(os.Stderr, " %s %d\n", styleDim.Render("query variants"), variants)
}
if res.PagesLinked > 0 {
fmt.Fprintf(os.Stderr, " %s %d\n", styleDim.Render("deduped (linked)"), res.PagesLinked)
}
if res.PageErrors+res.AssetErrors > 0 {
fmt.Fprintf(os.Stderr, " %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors)
printFailures(res)