Tam Nguyen Duc
7dc3621ced
Two related changes for crawling faceted sites, where one path spawns thousands of ?q=.../?page=... URLs that all render the same page. The progress line was counting every written file, so the page number ran far ahead of the site's real size. It now shows distinct URL paths as "pages" and folds the query-string permutations into a separate "variants" count, so the live counter tracks real pages and is easy to read. Pages with identical bytes are now stored once. The first page with a given content is written normally; a later page with the same bytes becomes a hard link to it, so duplicate content never takes disk twice. Links still resolve because each variant keeps its own path. When hard links are unsupported the bytes are written, so correctness never depends on the link. The summary reports how many pages were deduped.
109 lines
3.1 KiB
Go
109 lines
3.1 KiB
Go
package clone
|
|
|
|
import (
|
|
"sync"
|
|
"sync/atomic"
|
|
)
|
|
|
|
// maxRecordedFailures caps how many individual failures Run keeps for the final
|
|
// report, so a huge broken site cannot grow the slice without bound. The error
|
|
// counters still count every failure.
|
|
const maxRecordedFailures = 100
|
|
|
|
// stats are the live counters of a run, read by the CLI's progress ticker.
|
|
type stats struct {
|
|
pages atomic.Int64 // page documents written (one per output file)
|
|
pagePaths atomic.Int64 // distinct URL paths among those, ignoring query
|
|
pagesLinked atomic.Int64 // pages stored as a hard link to identical content
|
|
assets atomic.Int64
|
|
pageErrors atomic.Int64
|
|
assetErrors atomic.Int64
|
|
skipped atomic.Int64 // robots-disallowed or out of budget
|
|
|
|
muPaths sync.Mutex
|
|
seenPath map[string]struct{}
|
|
|
|
muFail sync.Mutex
|
|
failures []Failure
|
|
}
|
|
|
|
// recordPage counts a freshly written page. Every write bumps pages; the first
|
|
// write for a given query-stripped path also bumps pagePaths, so the display can
|
|
// separate real pages from the query-string variants (?q=…, ?page=…) that a
|
|
// single path can spawn by the thousand on a faceted site. deduped marks a page
|
|
// whose bytes were stored as a hard link to identical content already on disk.
|
|
func (s *stats) recordPage(pathKey string, deduped bool) {
|
|
s.pages.Add(1)
|
|
if deduped {
|
|
s.pagesLinked.Add(1)
|
|
}
|
|
s.muPaths.Lock()
|
|
if s.seenPath == nil {
|
|
s.seenPath = make(map[string]struct{})
|
|
}
|
|
if _, ok := s.seenPath[pathKey]; !ok {
|
|
s.seenPath[pathKey] = struct{}{}
|
|
s.pagePaths.Add(1)
|
|
}
|
|
s.muPaths.Unlock()
|
|
}
|
|
|
|
// Failure is one thing that went wrong, kept for the end-of-run report so the
|
|
// errors are visible as a list rather than only as a count.
|
|
type Failure struct {
|
|
Kind string // "page" or "asset"
|
|
URL string
|
|
Referer string // the page that referenced it, when known
|
|
Reason string // e.g. "HTTP 403 Forbidden"
|
|
}
|
|
|
|
func (s *stats) recordFailure(f Failure) {
|
|
s.muFail.Lock()
|
|
if len(s.failures) < maxRecordedFailures {
|
|
s.failures = append(s.failures, f)
|
|
}
|
|
s.muFail.Unlock()
|
|
}
|
|
|
|
func (s *stats) recordedFailures() []Failure {
|
|
s.muFail.Lock()
|
|
defer s.muFail.Unlock()
|
|
out := make([]Failure, len(s.failures))
|
|
copy(out, s.failures)
|
|
return out
|
|
}
|
|
|
|
// Progress is a snapshot of a run for display. Pages is every page document
|
|
// written (it equals the count of HTML files on disk); PagePaths is how many
|
|
// distinct URL paths those represent once query strings are ignored. The
|
|
// difference, Pages-PagePaths, is the number of query-string variants.
|
|
type Progress struct {
|
|
Pages int64
|
|
PagePaths int64
|
|
PagesLinked int64
|
|
Assets int64
|
|
PageErrors int64
|
|
AssetErrors int64
|
|
Skipped int64
|
|
}
|
|
|
|
func (s *stats) snapshot() Progress {
|
|
return Progress{
|
|
Pages: s.pages.Load(),
|
|
PagePaths: s.pagePaths.Load(),
|
|
PagesLinked: s.pagesLinked.Load(),
|
|
Assets: s.assets.Load(),
|
|
PageErrors: s.pageErrors.Load(),
|
|
AssetErrors: s.assetErrors.Load(),
|
|
Skipped: s.skipped.Load(),
|
|
}
|
|
}
|
|
|
|
// Result is the final outcome returned by Run.
|
|
type Result struct {
|
|
Progress
|
|
OutDir string
|
|
// Failures is a capped sample of what went wrong, for the final report.
|
|
Failures []Failure
|
|
}
|