Duc-Tam Nguyen
249761f29f
A developer.apple.com crawl came back at 19 GB, 18 of it assets, most of that the site's own videos, .dmg/.pkg installers, and PDF manuals pulled from a couple dozen hosts including unrelated third parties. None of it helps read the docs offline. This changes what kage localizes by default. Bulk media, installers, archives, and PDFs are left pointing at their live URL instead of downloaded. The decision is made from the URL alone, so the rewritten HTML simply keeps the remote link. --keep-media restores the old behavior and --skip-ext adds more extensions to leave remote. Assets are localized only from the seed's registrable domain by default, so www.apple.com and images.apple.com still come along but a separate brand domain or an off-topic third party does not. --all-asset-hosts goes back to downloading from any host. The size cap was also truncating instead of skipping: it wrapped the body in a LimitReader, so an over-cap file was saved as exactly the first N MB of itself, a corrupt fragment that would never play or run. On the apple crawl that was around a gigabyte of half-downloaded WWDC videos. kage now checks the response size and leaves an over-cap asset out of the mirror.
112 lines
3.3 KiB
Go
112 lines
3.3 KiB
Go
package clone
|
|
|
|
import (
|
|
"sync"
|
|
"sync/atomic"
|
|
)
|
|
|
|
// maxRecordedFailures caps how many individual failures Run keeps for the final
|
|
// report, so a huge broken site cannot grow the slice without bound. The error
|
|
// counters still count every failure.
|
|
const maxRecordedFailures = 100
|
|
|
|
// stats are the live counters of a run, read by the CLI's progress ticker.
|
|
type stats struct {
|
|
pages atomic.Int64 // page documents written (one per output file)
|
|
pagePaths atomic.Int64 // distinct URL paths among those, ignoring query
|
|
pagesLinked atomic.Int64 // pages stored as a hard link to identical content
|
|
assets atomic.Int64
|
|
assetSkipped atomic.Int64 // assets left on the live web (over the size cap)
|
|
pageErrors atomic.Int64
|
|
assetErrors atomic.Int64
|
|
skipped atomic.Int64 // robots-disallowed or out of budget
|
|
|
|
muPaths sync.Mutex
|
|
seenPath map[string]struct{}
|
|
|
|
muFail sync.Mutex
|
|
failures []Failure
|
|
}
|
|
|
|
// recordPage counts a freshly written page. Every write bumps pages; the first
|
|
// write for a given query-stripped path also bumps pagePaths, so the display can
|
|
// separate real pages from the query-string variants (?q=…, ?page=…) that a
|
|
// single path can spawn by the thousand on a faceted site. deduped marks a page
|
|
// whose bytes were stored as a hard link to identical content already on disk.
|
|
func (s *stats) recordPage(pathKey string, deduped bool) {
|
|
s.pages.Add(1)
|
|
if deduped {
|
|
s.pagesLinked.Add(1)
|
|
}
|
|
s.muPaths.Lock()
|
|
if s.seenPath == nil {
|
|
s.seenPath = make(map[string]struct{})
|
|
}
|
|
if _, ok := s.seenPath[pathKey]; !ok {
|
|
s.seenPath[pathKey] = struct{}{}
|
|
s.pagePaths.Add(1)
|
|
}
|
|
s.muPaths.Unlock()
|
|
}
|
|
|
|
// Failure is one thing that went wrong, kept for the end-of-run report so the
|
|
// errors are visible as a list rather than only as a count.
|
|
type Failure struct {
|
|
Kind string // "page" or "asset"
|
|
URL string
|
|
Referer string // the page that referenced it, when known
|
|
Reason string // e.g. "HTTP 403 Forbidden"
|
|
}
|
|
|
|
func (s *stats) recordFailure(f Failure) {
|
|
s.muFail.Lock()
|
|
if len(s.failures) < maxRecordedFailures {
|
|
s.failures = append(s.failures, f)
|
|
}
|
|
s.muFail.Unlock()
|
|
}
|
|
|
|
func (s *stats) recordedFailures() []Failure {
|
|
s.muFail.Lock()
|
|
defer s.muFail.Unlock()
|
|
out := make([]Failure, len(s.failures))
|
|
copy(out, s.failures)
|
|
return out
|
|
}
|
|
|
|
// Progress is a snapshot of a run for display. Pages is every page document
|
|
// written (it equals the count of HTML files on disk); PagePaths is how many
|
|
// distinct URL paths those represent once query strings are ignored. The
|
|
// difference, Pages-PagePaths, is the number of query-string variants.
|
|
type Progress struct {
|
|
Pages int64
|
|
PagePaths int64
|
|
PagesLinked int64
|
|
Assets int64
|
|
AssetSkipped int64
|
|
PageErrors int64
|
|
AssetErrors int64
|
|
Skipped int64
|
|
}
|
|
|
|
func (s *stats) snapshot() Progress {
|
|
return Progress{
|
|
Pages: s.pages.Load(),
|
|
PagePaths: s.pagePaths.Load(),
|
|
PagesLinked: s.pagesLinked.Load(),
|
|
Assets: s.assets.Load(),
|
|
AssetSkipped: s.assetSkipped.Load(),
|
|
PageErrors: s.pageErrors.Load(),
|
|
AssetErrors: s.assetErrors.Load(),
|
|
Skipped: s.skipped.Load(),
|
|
}
|
|
}
|
|
|
|
// Result is the final outcome returned by Run.
|
|
type Result struct {
|
|
Progress
|
|
OutDir string
|
|
// Failures is a capped sample of what went wrong, for the final report.
|
|
Failures []Failure
|
|
}
|