diff --git a/CHANGELOG.md b/CHANGELOG.md index fec618c..2a5a5a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ All notable changes to kage are recorded here. The format follows ## [Unreleased] +### Added + +- Identical pages are now stored once. When a rendered page's bytes match a page + already written, kage stores it as a hard link to the first copy instead of a + second full file. This collapses the duplicate content a faceted site spawns + when many `?q=…`/`?page=…` URLs all render the same page. The final summary + reports how many pages were deduped this way. + +### Changed + +- The progress line now counts real pages. "pages" is the number of distinct URL + paths written, and the query-string variants that one path can spawn by the + thousand are shown separately as "variants", so the live counter tracks the + site's real size instead of being inflated by `?q=…` permutations. + ## [0.2.1] - 2026-06-15 ### Added diff --git a/cli/clone.go b/cli/clone.go index 1133223..f9efbdb 100644 --- a/cli/clone.go +++ b/cli/clone.go @@ -172,18 +172,30 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error { return nil } -// progressLine renders the single-line live counter. +// progressLine renders the single-line live counter. "pages" is the count of +// real pages (distinct paths); when a faceted site spawns query-string variants +// they are shown separately so the page number stays easy to read. func progressLine(p clone.Progress) string { + if variants := p.Pages - p.PagePaths; variants > 0 { + return styleDim.Render(fmt.Sprintf("pages %d variants %d assets %d errors %d skipped %d", + p.PagePaths, variants, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped)) + } return styleDim.Render(fmt.Sprintf("pages %d assets %d errors %d skipped %d", - p.Pages, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped)) + p.PagePaths, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped)) } // printSummary prints the final tally and where the mirror landed. func printSummary(res clone.Result) { fmt.Fprintln(os.Stderr, styleOK.Render("done")+" "+styleTitle.Render(res.OutDir)) fmt.Fprintf(os.Stderr, " %s %d %s %d\n", - styleAccent.Render("pages"), res.Pages, + styleAccent.Render("pages"), res.PagePaths, styleAccent.Render("assets"), res.Assets) + if variants := res.Pages - res.PagePaths; variants > 0 { + fmt.Fprintf(os.Stderr, " %s %d\n", styleDim.Render("query variants"), variants) + } + if res.PagesLinked > 0 { + fmt.Fprintf(os.Stderr, " %s %d\n", styleDim.Render("deduped (linked)"), res.PagesLinked) + } if res.PageErrors+res.AssetErrors > 0 { fmt.Fprintf(os.Stderr, " %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors) printFailures(res) diff --git a/clone/cloner.go b/clone/cloner.go index 48486db..06dd69f 100644 --- a/clone/cloner.go +++ b/clone/cloner.go @@ -2,6 +2,7 @@ package clone import ( "context" + "crypto/sha256" "errors" "fmt" "io" @@ -45,6 +46,9 @@ type Cloner struct { wg sync.WaitGroup pageJobs chan pageItem assetJobs chan assetItem + + muContent sync.Mutex + seenContent map[string]string // sha-256 of page bytes -> first path written } type pageItem struct { @@ -66,19 +70,20 @@ func New(seed *url.URL, cfg Config, logf Logf) *Cloner { host := seed.Hostname() outRoot := cfg.HostDir(host) return &Cloner{ - cfg: cfg, - seed: seed, - seedHost: host, - outRoot: outRoot, - statePth: filepath.Join(outRoot, cfg.Reserved, "state.json"), - dl: asset.NewDownloader(cfg.UserAgent, cfg.Timeout, cfg.MaxAssetBytes), - httpC: &http.Client{Timeout: cfg.Timeout}, - robots: robots.AllowAll(), - front: newFrontier(), - logf: logf, - seenAssets: map[string]bool{}, - pageJobs: make(chan pageItem), - assetJobs: make(chan assetItem), + cfg: cfg, + seed: seed, + seedHost: host, + outRoot: outRoot, + statePth: filepath.Join(outRoot, cfg.Reserved, "state.json"), + dl: asset.NewDownloader(cfg.UserAgent, cfg.Timeout, cfg.MaxAssetBytes), + httpC: &http.Client{Timeout: cfg.Timeout}, + robots: robots.AllowAll(), + front: newFrontier(), + logf: logf, + seenAssets: map[string]bool{}, + seenContent: map[string]string{}, + pageJobs: make(chan pageItem), + assetJobs: make(chan assetItem), } } @@ -98,6 +103,19 @@ func (c *Cloner) assetKey(u *url.URL) string { return urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved) } +// pagePathKey is the identity of a page ignoring its query string, used to tell +// a real page apart from its ?q=…/?page=… variants for the progress display. +// Each variant writes its own file (so the crawl stays complete), but they all +// fold to one path here. +func (c *Cloner) pagePathKey(u *url.URL) string { + if u.RawQuery == "" { + return c.pageKey(u) + } + cp := *u + cp.RawQuery = "" + return c.pageKey(&cp) +} + // Run executes the clone until the frontier drains, MaxPages is hit, or ctx is // cancelled (which flushes the resume state). It returns the final Result. func (c *Cloner) Run(ctx context.Context) (Result, error) { @@ -277,12 +295,13 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) { c.failPage(j.u.String(), fmt.Errorf("render html: %w", err)) return } - if err := c.writeFile(localFile, []byte(buf.String())); err != nil { + deduped, err := c.writePage(localFile, []byte(buf.String())) + if err != nil { c.failPage(j.u.String(), fmt.Errorf("write %s: %w", localFile, err)) return } c.front.markVisited(key) - c.stats.pages.Add(1) + c.stats.recordPage(c.pagePathKey(j.u), deduped) } // processAsset downloads one asset, rewriting CSS references on the way, and @@ -410,6 +429,49 @@ func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) { }() } +// writePage writes a rendered page, deduping by content. The first page with a +// given byte content is written normally; a later page with identical bytes is +// stored as a hard link to that first file, so the same content never occupies +// disk twice (a faceted site whose ?q=… variants all render the same page is the +// motivating case). It reports whether this write was deduped. If hard links are +// unsupported, it falls back to writing the bytes, so correctness never depends +// on the link succeeding. +func (c *Cloner) writePage(relSlash string, data []byte) (bool, error) { + sum := sha256.Sum256(data) + h := string(sum[:]) + + c.muContent.Lock() + canon, seen := c.seenContent[h] + if !seen { + c.seenContent[h] = relSlash + } + c.muContent.Unlock() + + if seen && canon != relSlash { + if err := c.linkFile(canon, relSlash); err == nil { + return true, nil + } + // The canonical file may not be on disk yet (a concurrent first write) + // or the filesystem may not support links; write the bytes instead. + } + return false, c.writeFile(relSlash, data) +} + +// linkFile hard-links the already-written canonical file to target, replacing +// any file already at target. Both paths are confined to the mirror root. +func (c *Cloner) linkFile(canonSlash, targetSlash string) error { + canonFull := filepath.Join(c.outRoot, filepath.FromSlash(canonSlash)) + targetFull := filepath.Join(c.outRoot, filepath.FromSlash(targetSlash)) + if !strings.HasPrefix(targetFull, filepath.Clean(c.outRoot)+string(os.PathSeparator)) { + return fmt.Errorf("refusing to link outside mirror root: %s", targetSlash) + } + if err := os.MkdirAll(filepath.Dir(targetFull), 0o755); err != nil { + return err + } + _ = os.Remove(targetFull) + return os.Link(canonFull, targetFull) +} + // writeFile writes data to a slash path relative to the mirror root, creating // parent directories. The path is cleaned so it can never escape the root. func (c *Cloner) writeFile(relSlash string, data []byte) error { diff --git a/clone/dedup_test.go b/clone/dedup_test.go new file mode 100644 index 0000000..89dca3f --- /dev/null +++ b/clone/dedup_test.go @@ -0,0 +1,69 @@ +package clone + +import ( + "os" + "path/filepath" + "testing" +) + +// TestWritePageDedup checks that identical page bytes are stored once and shared +// by a hard link, while different bytes are written as their own file. +func TestWritePageDedup(t *testing.T) { + dir := t.TempDir() + c := &Cloner{outRoot: dir, seenContent: map[string]string{}} + + body := []byte("same page") + + if deduped, err := c.writePage("a/index.html", body); err != nil || deduped { + t.Fatalf("first write: deduped=%v err=%v, want false/nil", deduped, err) + } + if deduped, err := c.writePage("b/index.html", body); err != nil || !deduped { + t.Fatalf("second identical write: deduped=%v err=%v, want true/nil", deduped, err) + } + if deduped, err := c.writePage("c/index.html", []byte("other")); err != nil || deduped { + t.Fatalf("third different write: deduped=%v err=%v, want false/nil", deduped, err) + } + + // The two identical pages must be the same file on disk (one inode). + fa, err := os.Stat(filepath.Join(dir, "a/index.html")) + if err != nil { + t.Fatal(err) + } + fb, err := os.Stat(filepath.Join(dir, "b/index.html")) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(fa, fb) { + t.Error("identical pages were not hard-linked to the same file") + } + + // The different page must stand alone with its own bytes. + got, err := os.ReadFile(filepath.Join(dir, "c/index.html")) + if err != nil { + t.Fatal(err) + } + if string(got) != "other" { + t.Errorf("third page content = %q", got) + } +} + +// TestRecordPageCounts checks that pages counts every write, pagePaths counts +// distinct query-stripped paths, and pagesLinked counts deduped writes. +func TestRecordPageCounts(t *testing.T) { + var s stats + s.recordPage("showcase/index.html", false) + s.recordPage("showcase/index.html", true) // a ?q= variant of the same path + s.recordPage("showcase/index.html", true) + s.recordPage("about/index.html", false) + + p := s.snapshot() + if p.Pages != 4 { + t.Errorf("Pages = %d, want 4", p.Pages) + } + if p.PagePaths != 2 { + t.Errorf("PagePaths = %d, want 2", p.PagePaths) + } + if p.PagesLinked != 2 { + t.Errorf("PagesLinked = %d, want 2", p.PagesLinked) + } +} diff --git a/clone/stats.go b/clone/stats.go index c153640..93aff68 100644 --- a/clone/stats.go +++ b/clone/stats.go @@ -12,16 +12,42 @@ const maxRecordedFailures = 100 // stats are the live counters of a run, read by the CLI's progress ticker. type stats struct { - pages atomic.Int64 + pages atomic.Int64 // page documents written (one per output file) + pagePaths atomic.Int64 // distinct URL paths among those, ignoring query + pagesLinked atomic.Int64 // pages stored as a hard link to identical content assets atomic.Int64 pageErrors atomic.Int64 assetErrors atomic.Int64 skipped atomic.Int64 // robots-disallowed or out of budget + muPaths sync.Mutex + seenPath map[string]struct{} + muFail sync.Mutex failures []Failure } +// recordPage counts a freshly written page. Every write bumps pages; the first +// write for a given query-stripped path also bumps pagePaths, so the display can +// separate real pages from the query-string variants (?q=…, ?page=…) that a +// single path can spawn by the thousand on a faceted site. deduped marks a page +// whose bytes were stored as a hard link to identical content already on disk. +func (s *stats) recordPage(pathKey string, deduped bool) { + s.pages.Add(1) + if deduped { + s.pagesLinked.Add(1) + } + s.muPaths.Lock() + if s.seenPath == nil { + s.seenPath = make(map[string]struct{}) + } + if _, ok := s.seenPath[pathKey]; !ok { + s.seenPath[pathKey] = struct{}{} + s.pagePaths.Add(1) + } + s.muPaths.Unlock() +} + // Failure is one thing that went wrong, kept for the end-of-run report so the // errors are visible as a list rather than only as a count. type Failure struct { @@ -47,9 +73,14 @@ func (s *stats) recordedFailures() []Failure { return out } -// Progress is a snapshot of a run for display. +// Progress is a snapshot of a run for display. Pages is every page document +// written (it equals the count of HTML files on disk); PagePaths is how many +// distinct URL paths those represent once query strings are ignored. The +// difference, Pages-PagePaths, is the number of query-string variants. type Progress struct { Pages int64 + PagePaths int64 + PagesLinked int64 Assets int64 PageErrors int64 AssetErrors int64 @@ -59,6 +90,8 @@ type Progress struct { func (s *stats) snapshot() Progress { return Progress{ Pages: s.pages.Load(), + PagePaths: s.pagePaths.Load(), + PagesLinked: s.pagesLinked.Load(), Assets: s.assets.Load(), PageErrors: s.pageErrors.Load(), AssetErrors: s.assetErrors.Load(),