Count real pages and store identical pages once (#20)

Two related changes for crawling faceted sites, where one path spawns thousands of ?q=.../?page=... URLs that all render the same page. The progress line was counting every written file, so the page number ran far ahead of the site's real size. It now shows distinct URL paths as "pages" and folds the query-string permutations into a separate "variants" count, so the live counter tracks real pages and is easy to read. Pages with identical bytes are now stored once. The first page with a given content is written normally; a later page with the same bytes becomes a hard link to it, so duplicate content never takes disk twice. Links still resolve because each variant keeps its own path. When hard links are unsupported the bytes are written, so correctness never depends on the link. The summary reports how many pages were deduped.
2026-06-15 19:07:49 +07:00
parent 8833a2b8f8
commit 7dc3621ced
5 changed files with 211 additions and 20 deletions
@@ -6,6 +6,21 @@ All notable changes to kage are recorded here. The format follows

 ## [Unreleased]

+### Added
+
+- Identical pages are now stored once. When a rendered page's bytes match a page
+  already written, kage stores it as a hard link to the first copy instead of a
+  second full file. This collapses the duplicate content a faceted site spawns
+  when many `?q=…`/`?page=…` URLs all render the same page. The final summary
+  reports how many pages were deduped this way.
+
+### Changed
+
+- The progress line now counts real pages. "pages" is the number of distinct URL
+  paths written, and the query-string variants that one path can spawn by the
+  thousand are shown separately as "variants", so the live counter tracks the
+  site's real size instead of being inflated by `?q=…` permutations.
+
 ## [0.2.1] - 2026-06-15

 ### Added
@@ -172,18 +172,30 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
 	return nil
 }

-// progressLine renders the single-line live counter.
+// progressLine renders the single-line live counter. "pages" is the count of
+// real pages (distinct paths); when a faceted site spawns query-string variants
+// they are shown separately so the page number stays easy to read.
 func progressLine(p clone.Progress) string {
+	if variants := p.Pages - p.PagePaths; variants > 0 {
+		return styleDim.Render(fmt.Sprintf("pages %d  variants %d  assets %d  errors %d  skipped %d",
+			p.PagePaths, variants, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
+	}
 	return styleDim.Render(fmt.Sprintf("pages %d  assets %d  errors %d  skipped %d",
-		p.Pages, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
+		p.PagePaths, p.Assets, p.PageErrors+p.AssetErrors, p.Skipped))
 }

 // printSummary prints the final tally and where the mirror landed.
 func printSummary(res clone.Result) {
 	fmt.Fprintln(os.Stderr, styleOK.Render("done")+" "+styleTitle.Render(res.OutDir))
 	fmt.Fprintf(os.Stderr, "  %s %d   %s %d\n",
-		styleAccent.Render("pages"), res.Pages,
+		styleAccent.Render("pages"), res.PagePaths,
 		styleAccent.Render("assets"), res.Assets)
+	if variants := res.Pages - res.PagePaths; variants > 0 {
+		fmt.Fprintf(os.Stderr, "  %s %d\n", styleDim.Render("query variants"), variants)
+	}
+	if res.PagesLinked > 0 {
+		fmt.Fprintf(os.Stderr, "  %s %d\n", styleDim.Render("deduped (linked)"), res.PagesLinked)
+	}
 	if res.PageErrors+res.AssetErrors > 0 {
 		fmt.Fprintf(os.Stderr, "  %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors)
 		printFailures(res)
@@ -2,6 +2,7 @@ package clone

 import (
 	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io"
@@ -45,6 +46,9 @@ type Cloner struct {
 	wg         sync.WaitGroup
 	pageJobs   chan pageItem
 	assetJobs  chan assetItem
+
+	muContent   sync.Mutex
+	seenContent map[string]string // sha-256 of page bytes -> first path written
 }

 type pageItem struct {
@@ -66,19 +70,20 @@ func New(seed *url.URL, cfg Config, logf Logf) *Cloner {
 	host := seed.Hostname()
 	outRoot := cfg.HostDir(host)
 	return &Cloner{
-		cfg:        cfg,
-		seed:       seed,
-		seedHost:   host,
-		outRoot:    outRoot,
-		statePth:   filepath.Join(outRoot, cfg.Reserved, "state.json"),
-		dl:         asset.NewDownloader(cfg.UserAgent, cfg.Timeout, cfg.MaxAssetBytes),
-		httpC:      &http.Client{Timeout: cfg.Timeout},
-		robots:     robots.AllowAll(),
-		front:      newFrontier(),
-		logf:       logf,
-		seenAssets: map[string]bool{},
-		pageJobs:   make(chan pageItem),
-		assetJobs:  make(chan assetItem),
+		cfg:         cfg,
+		seed:        seed,
+		seedHost:    host,
+		outRoot:     outRoot,
+		statePth:    filepath.Join(outRoot, cfg.Reserved, "state.json"),
+		dl:          asset.NewDownloader(cfg.UserAgent, cfg.Timeout, cfg.MaxAssetBytes),
+		httpC:       &http.Client{Timeout: cfg.Timeout},
+		robots:      robots.AllowAll(),
+		front:       newFrontier(),
+		logf:        logf,
+		seenAssets:  map[string]bool{},
+		seenContent: map[string]string{},
+		pageJobs:    make(chan pageItem),
+		assetJobs:   make(chan assetItem),
 	}
 }

@@ -98,6 +103,19 @@ func (c *Cloner) assetKey(u *url.URL) string {
 	return urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved)
 }

+// pagePathKey is the identity of a page ignoring its query string, used to tell
+// a real page apart from its ?q=…/?page=… variants for the progress display.
+// Each variant writes its own file (so the crawl stays complete), but they all
+// fold to one path here.
+func (c *Cloner) pagePathKey(u *url.URL) string {
+	if u.RawQuery == "" {
+		return c.pageKey(u)
+	}
+	cp := *u
+	cp.RawQuery = ""
+	return c.pageKey(&cp)
+}
+
 // Run executes the clone until the frontier drains, MaxPages is hit, or ctx is
 // cancelled (which flushes the resume state). It returns the final Result.
 func (c *Cloner) Run(ctx context.Context) (Result, error) {
@@ -277,12 +295,13 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 		c.failPage(j.u.String(), fmt.Errorf("render html: %w", err))
 		return
 	}
-	if err := c.writeFile(localFile, []byte(buf.String())); err != nil {
+	deduped, err := c.writePage(localFile, []byte(buf.String()))
+	if err != nil {
 		c.failPage(j.u.String(), fmt.Errorf("write %s: %w", localFile, err))
 		return
 	}
 	c.front.markVisited(key)
-	c.stats.pages.Add(1)
+	c.stats.recordPage(c.pagePathKey(j.u), deduped)
 }

 // processAsset downloads one asset, rewriting CSS references on the way, and
@@ -410,6 +429,49 @@ func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) {
 	}()
 }

+// writePage writes a rendered page, deduping by content. The first page with a
+// given byte content is written normally; a later page with identical bytes is
+// stored as a hard link to that first file, so the same content never occupies
+// disk twice (a faceted site whose ?q=… variants all render the same page is the
+// motivating case). It reports whether this write was deduped. If hard links are
+// unsupported, it falls back to writing the bytes, so correctness never depends
+// on the link succeeding.
+func (c *Cloner) writePage(relSlash string, data []byte) (bool, error) {
+	sum := sha256.Sum256(data)
+	h := string(sum[:])
+
+	c.muContent.Lock()
+	canon, seen := c.seenContent[h]
+	if !seen {
+		c.seenContent[h] = relSlash
+	}
+	c.muContent.Unlock()
+
+	if seen && canon != relSlash {
+		if err := c.linkFile(canon, relSlash); err == nil {
+			return true, nil
+		}
+		// The canonical file may not be on disk yet (a concurrent first write)
+		// or the filesystem may not support links; write the bytes instead.
+	}
+	return false, c.writeFile(relSlash, data)
+}
+
+// linkFile hard-links the already-written canonical file to target, replacing
+// any file already at target. Both paths are confined to the mirror root.
+func (c *Cloner) linkFile(canonSlash, targetSlash string) error {
+	canonFull := filepath.Join(c.outRoot, filepath.FromSlash(canonSlash))
+	targetFull := filepath.Join(c.outRoot, filepath.FromSlash(targetSlash))
+	if !strings.HasPrefix(targetFull, filepath.Clean(c.outRoot)+string(os.PathSeparator)) {
+		return fmt.Errorf("refusing to link outside mirror root: %s", targetSlash)
+	}
+	if err := os.MkdirAll(filepath.Dir(targetFull), 0o755); err != nil {
+		return err
+	}
+	_ = os.Remove(targetFull)
+	return os.Link(canonFull, targetFull)
+}
+
 // writeFile writes data to a slash path relative to the mirror root, creating
 // parent directories. The path is cleaned so it can never escape the root.
 func (c *Cloner) writeFile(relSlash string, data []byte) error {
@@ -0,0 +1,69 @@
+package clone
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestWritePageDedup checks that identical page bytes are stored once and shared
+// by a hard link, while different bytes are written as their own file.
+func TestWritePageDedup(t *testing.T) {
+	dir := t.TempDir()
+	c := &Cloner{outRoot: dir, seenContent: map[string]string{}}
+
+	body := []byte("<html><body>same page</body></html>")
+
+	if deduped, err := c.writePage("a/index.html", body); err != nil || deduped {
+		t.Fatalf("first write: deduped=%v err=%v, want false/nil", deduped, err)
+	}
+	if deduped, err := c.writePage("b/index.html", body); err != nil || !deduped {
+		t.Fatalf("second identical write: deduped=%v err=%v, want true/nil", deduped, err)
+	}
+	if deduped, err := c.writePage("c/index.html", []byte("<html>other</html>")); err != nil || deduped {
+		t.Fatalf("third different write: deduped=%v err=%v, want false/nil", deduped, err)
+	}
+
+	// The two identical pages must be the same file on disk (one inode).
+	fa, err := os.Stat(filepath.Join(dir, "a/index.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	fb, err := os.Stat(filepath.Join(dir, "b/index.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !os.SameFile(fa, fb) {
+		t.Error("identical pages were not hard-linked to the same file")
+	}
+
+	// The different page must stand alone with its own bytes.
+	got, err := os.ReadFile(filepath.Join(dir, "c/index.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != "<html>other</html>" {
+		t.Errorf("third page content = %q", got)
+	}
+}
+
+// TestRecordPageCounts checks that pages counts every write, pagePaths counts
+// distinct query-stripped paths, and pagesLinked counts deduped writes.
+func TestRecordPageCounts(t *testing.T) {
+	var s stats
+	s.recordPage("showcase/index.html", false)
+	s.recordPage("showcase/index.html", true) // a ?q= variant of the same path
+	s.recordPage("showcase/index.html", true)
+	s.recordPage("about/index.html", false)
+
+	p := s.snapshot()
+	if p.Pages != 4 {
+		t.Errorf("Pages = %d, want 4", p.Pages)
+	}
+	if p.PagePaths != 2 {
+		t.Errorf("PagePaths = %d, want 2", p.PagePaths)
+	}
+	if p.PagesLinked != 2 {
+		t.Errorf("PagesLinked = %d, want 2", p.PagesLinked)
+	}
+}
@@ -12,16 +12,42 @@ const maxRecordedFailures = 100

 // stats are the live counters of a run, read by the CLI's progress ticker.
 type stats struct {
-	pages       atomic.Int64
+	pages       atomic.Int64 // page documents written (one per output file)
+	pagePaths   atomic.Int64 // distinct URL paths among those, ignoring query
+	pagesLinked atomic.Int64 // pages stored as a hard link to identical content
 	assets      atomic.Int64
 	pageErrors  atomic.Int64
 	assetErrors atomic.Int64
 	skipped     atomic.Int64 // robots-disallowed or out of budget

+	muPaths  sync.Mutex
+	seenPath map[string]struct{}
+
 	muFail   sync.Mutex
 	failures []Failure
 }

+// recordPage counts a freshly written page. Every write bumps pages; the first
+// write for a given query-stripped path also bumps pagePaths, so the display can
+// separate real pages from the query-string variants (?q=…, ?page=…) that a
+// single path can spawn by the thousand on a faceted site. deduped marks a page
+// whose bytes were stored as a hard link to identical content already on disk.
+func (s *stats) recordPage(pathKey string, deduped bool) {
+	s.pages.Add(1)
+	if deduped {
+		s.pagesLinked.Add(1)
+	}
+	s.muPaths.Lock()
+	if s.seenPath == nil {
+		s.seenPath = make(map[string]struct{})
+	}
+	if _, ok := s.seenPath[pathKey]; !ok {
+		s.seenPath[pathKey] = struct{}{}
+		s.pagePaths.Add(1)
+	}
+	s.muPaths.Unlock()
+}
+
 // Failure is one thing that went wrong, kept for the end-of-run report so the
 // errors are visible as a list rather than only as a count.
 type Failure struct {
@@ -47,9 +73,14 @@ func (s *stats) recordedFailures() []Failure {
 	return out
 }

-// Progress is a snapshot of a run for display.
+// Progress is a snapshot of a run for display. Pages is every page document
+// written (it equals the count of HTML files on disk); PagePaths is how many
+// distinct URL paths those represent once query strings are ignored. The
+// difference, Pages-PagePaths, is the number of query-string variants.
 type Progress struct {
 	Pages       int64
+	PagePaths   int64
+	PagesLinked int64
 	Assets      int64
 	PageErrors  int64
 	AssetErrors int64
@@ -59,6 +90,8 @@ type Progress struct {
 func (s *stats) snapshot() Progress {
 	return Progress{
 		Pages:       s.pages.Load(),
+		PagePaths:   s.pagePaths.Load(),
+		PagesLinked: s.pagesLinked.Load(),
 		Assets:      s.assets.Load(),
 		PageErrors:  s.pageErrors.Load(),
 		AssetErrors: s.assetErrors.Load(),