diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b94bc5..127126f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,18 @@ All notable changes to kage are recorded here. The format follows ### Changed +- Clones no longer pull a site's bulk downloads into the mirror by default. Video + and audio, installers and disk images (`.dmg`, `.pkg`, `.exe`, `.msi`, ...), + archives, and PDFs are left pointing at their live URL instead of downloaded, + because they are rarely needed to read a site offline yet routinely make up + most of its bytes (a developer.apple.com crawl was 18 of 19 GB of such assets). + Page-rendering assets (images, fonts, CSS) are unaffected. `--keep-media` + restores the old behavior, and `--skip-ext .foo` leaves more extensions remote. +- Assets are localized only from the seed's own registrable domain by default. + developer.apple.com still pulls from www.apple.com and images.apple.com, but a + separate brand domain or an unrelated third party (an embedded tracker, an + off-topic CDN) is left on the live web rather than mirrored. `--all-asset-hosts` + restores downloading assets from any host. - The progress line now counts real pages. "pages" is the number of distinct URL paths written, and the query-string variants that one path can spawn by the thousand are shown separately as "variants", so the live counter tracks the @@ -41,6 +53,13 @@ All notable changes to kage are recorded here. The format follows ### Fixed +- An asset larger than the size cap (`--max-asset-mb`, 25 by default) is now + skipped instead of being truncated to a corrupt fragment. The cap was a + `LimitReader`, so an over-size file was saved as exactly the first N MB of + itself: a broken video or installer that wasted disk and would never play or + run. kage now checks the response size and leaves an over-cap asset out of the + mirror entirely, pointing at its live URL. On a developer.apple.com crawl this + was around a gigabyte of truncated WWDC videos and `.dmg` installers. - An asset URL whose query string carries a raw space is now requested with the space percent-encoded, so the server gets a valid request instead of rejecting it. Real sites bust a stylesheet's cache with a date, producing an href like diff --git a/asset/download.go b/asset/download.go index 6d6958b..b8c2112 100644 --- a/asset/download.go +++ b/asset/download.go @@ -42,6 +42,13 @@ type Result struct { IsCSS bool } +// ErrTooLarge reports that an asset exceeds the size cap and was skipped without +// being saved. It is deliberately a skip, not a download failure: the caller +// leaves the asset out of the mirror rather than writing a truncated fragment of +// it, so a 500 MB installer or video never bloats the archive with a corrupt +// quarter of itself. +var ErrTooLarge = errors.New("asset over size cap") + // StatusError reports a non-2xx HTTP response. It carries the code so callers // can render a clear message ("HTTP 403 Forbidden") and decide whether a retry // is worthwhile, without the URL baked in (the caller already has it). @@ -109,14 +116,26 @@ func (d *Downloader) try(ctx context.Context, u *url.URL, referer string) (*Resu if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, &StatusError{Code: resp.StatusCode} } + // Skip an over-cap asset instead of truncating it. A Content-Length lets us + // bail before reading a byte; otherwise we read one byte past the cap and, if + // the body really is larger, discard what we have. Either way nothing partial + // reaches disk. + if d.MaxBytes > 0 && resp.ContentLength > d.MaxBytes { + return nil, ErrTooLarge + } var r io.Reader = resp.Body if d.MaxBytes > 0 { - r = io.LimitReader(resp.Body, d.MaxBytes) + // Read at most one byte past the cap so a body with no (or a lying) + // Content-Length cannot stream gigabytes into memory before we notice. + r = io.LimitReader(resp.Body, d.MaxBytes+1) } body, err := io.ReadAll(r) if err != nil { return nil, err } + if d.MaxBytes > 0 && int64(len(body)) > d.MaxBytes { + return nil, ErrTooLarge + } ct := resp.Header.Get("Content-Type") return &Result{ Body: body, @@ -142,6 +161,9 @@ func transient(err error) bool { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return false } + if errors.Is(err, ErrTooLarge) { + return false + } var se *StatusError if errors.As(err, &se) { switch se.Code { diff --git a/asset/download_test.go b/asset/download_test.go index d74ffb9..83fd70d 100644 --- a/asset/download_test.go +++ b/asset/download_test.go @@ -94,6 +94,69 @@ func TestGetGivesUpAfterRetries(t *testing.T) { } } +func TestGetSkipsOverCapByContentLength(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + body := make([]byte, 4096) // declared via Content-Length + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write(body) + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 1024) // cap below the body + u, _ := url.Parse(srv.URL + "/clip.mp4") + _, err := d.Get(context.Background(), u, "") + if !errors.Is(err, ErrTooLarge) { + t.Fatalf("got %v; want ErrTooLarge", err) + } + if hits != 1 { + t.Errorf("an over-cap asset should not be retried; server saw %d hits", hits) + } +} + +func TestGetSkipsOverCapWithoutContentLength(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // A chunked response carries no Content-Length, so the cap is only known + // after reading past it. + w.Header().Set("Content-Type", "application/octet-stream") + fl, _ := w.(http.Flusher) + chunk := make([]byte, 512) + for i := 0; i < 8; i++ { + _, _ = w.Write(chunk) + if fl != nil { + fl.Flush() + } + } + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 1024) + u, _ := url.Parse(srv.URL + "/stream.bin") + _, err := d.Get(context.Background(), u, "") + if !errors.Is(err, ErrTooLarge) { + t.Fatalf("got %v; want ErrTooLarge", err) + } +} + +func TestGetKeepsUnderCap(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write([]byte("small")) + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 1024) + u, _ := url.Parse(srv.URL + "/logo.png") + res, err := d.Get(context.Background(), u, "") + if err != nil { + t.Fatalf("under-cap asset: %v", err) + } + if string(res.Body) != "small" { + t.Errorf("body = %q; want %q", res.Body, "small") + } +} + func TestTransientClassification(t *testing.T) { transientCodes := []int{403, 408, 425, 429, 500, 502, 503} for _, c := range transientCodes { diff --git a/cli/clone.go b/cli/clone.go index f9efbdb..280e372 100644 --- a/cli/clone.go +++ b/cli/clone.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "strings" "time" "github.com/spf13/cobra" @@ -15,33 +16,36 @@ import ( // cloneFlags holds the parsed flag values for one invocation. type cloneFlags struct { - out string - reserved string - workers int - assetWorkers int - browserPages int - maxPages int - maxDepth int - traversal string - maxAssetMB int64 - timeout time.Duration - settle time.Duration - renderTO time.Duration - scroll bool - userAgent string - subdomains bool - scopePrefix string - exclude []string - noRobots bool - noSitemap bool - headful bool - keepNoscript bool - chromeBin string - controlURL string - noResume bool - refresh bool - force bool - quiet bool + out string + reserved string + workers int + assetWorkers int + browserPages int + maxPages int + maxDepth int + traversal string + maxAssetMB int64 + keepMedia bool + skipExt []string + allAssetHosts bool + timeout time.Duration + settle time.Duration + renderTO time.Duration + scroll bool + userAgent string + subdomains bool + scopePrefix string + exclude []string + noRobots bool + noSitemap bool + headful bool + keepNoscript bool + chromeBin string + controlURL string + noResume bool + refresh bool + force bool + quiet bool } func newCloneCmd() *cobra.Command { @@ -66,7 +70,10 @@ func newCloneCmd() *cobra.Command { fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "stop after N pages (0 = unlimited)") fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)") fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order: bfs or dfs") - fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB") + fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB (left on the live web)") + fs.BoolVar(&f.keepMedia, "keep-media", false, "download bulk media, installers, and PDFs instead of leaving them remote") + fs.StringSliceVar(&f.skipExt, "skip-ext", nil, "extra asset extensions to leave remote, e.g. .svg (repeatable)") + fs.BoolVar(&f.allAssetHosts, "all-asset-hosts", false, "localize assets from any host, not just the seed's domain") fs.DurationVar(&f.timeout, "timeout", 30*time.Second, "per-request timeout") fs.DurationVar(&f.settle, "settle", 1500*time.Millisecond, "network-idle quiet period before snapshot") fs.DurationVar(&f.renderTO, "render-timeout", 30*time.Second, "hard cap per page render") @@ -104,6 +111,23 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error { cfg.MaxDepth = f.maxDepth cfg.Traversal = f.traversal cfg.MaxAssetBytes = f.maxAssetMB << 20 + cfg.AssetSameDomain = !f.allAssetHosts + if f.keepMedia { + cfg.SkipAssetExts = map[string]bool{} + } + for _, e := range f.skipExt { + e = strings.ToLower(strings.TrimSpace(e)) + if e == "" { + continue + } + if !strings.HasPrefix(e, ".") { + e = "." + e + } + if cfg.SkipAssetExts == nil { + cfg.SkipAssetExts = map[string]bool{} + } + cfg.SkipAssetExts[e] = true + } cfg.Timeout = f.timeout cfg.Settle = f.settle cfg.RenderTimeout = f.renderTO @@ -196,6 +220,9 @@ func printSummary(res clone.Result) { if res.PagesLinked > 0 { fmt.Fprintf(os.Stderr, " %s %d\n", styleDim.Render("deduped (linked)"), res.PagesLinked) } + if res.AssetSkipped > 0 { + fmt.Fprintf(os.Stderr, " %s %d\n", styleDim.Render("assets over cap (left remote)"), res.AssetSkipped) + } if res.PageErrors+res.AssetErrors > 0 { fmt.Fprintf(os.Stderr, " %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors) printFailures(res) diff --git a/clone/cloner.go b/clone/cloner.go index 06dd69f..086cd7d 100644 --- a/clone/cloner.go +++ b/clone/cloner.go @@ -278,6 +278,9 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) { } return u.String() // external page link stays on the live web default: // Asset + if !c.wantAsset(u) { + return u.String() // off-domain or bulk media: leave it on the live web + } c.enqueueAsset(ctx, u, j.u.String()) local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved) return urlx.Rel(fileDir, local) @@ -312,6 +315,13 @@ func (c *Cloner) processAsset(ctx context.Context, j assetItem) { } res, err := c.dl.Get(ctx, j.u, j.referer) if err != nil { + if errors.Is(err, asset.ErrTooLarge) { + // Over the size cap: leave it out rather than save a truncated + // fragment. Count it as skipped, not failed, so the run is clean. + c.stats.assetSkipped.Add(1) + c.logf("asset skipped (over %d MB): %s", c.cfg.MaxAssetBytes>>20, j.u.String()) + return + } c.failAsset(j.u.String(), j.referer, err) return } @@ -321,6 +331,9 @@ func (c *Cloner) processAsset(ctx context.Context, j assetItem) { if res.IsCSS { fileDir := urlx.Dir(localFile) cssSink := func(u *url.URL, _ urlx.Kind) string { + if !c.wantAsset(u) { + return u.String() // off-domain or bulk media: leave it on the live web + } c.enqueueAsset(ctx, u, j.u.String()) local := urlx.LocalPath(c.seedHost, u, urlx.Asset, c.cfg.Reserved) return urlx.Rel(fileDir, local) @@ -408,6 +421,22 @@ func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool { return true } +// wantAsset reports whether an asset should be downloaded and localized, or left +// pointing at its live URL. kage skips two classes by default: assets on a host +// outside the seed's registrable domain (a third-party tracker, an unrelated +// CDN), and bulk media, installers, and archives whose extension is in the skip +// set. Both are decisions a page worker can make from the URL alone, before any +// download, so the rewritten HTML simply keeps the remote link. +func (c *Cloner) wantAsset(u *url.URL) bool { + if c.cfg.AssetSameDomain && !urlx.SameRegistrableDomain(c.seed, u) { + return false + } + if c.cfg.SkipAssetExts[urlx.Ext(u)] { + return false + } + return true +} + // enqueueAsset offers an asset URL for download, deduping by canonical URL. func (c *Cloner) enqueueAsset(ctx context.Context, u *url.URL, referer string) { key := c.assetKey(u) diff --git a/clone/config.go b/clone/config.go index 7eef526..e419a2f 100644 --- a/clone/config.go +++ b/clone/config.go @@ -36,6 +36,16 @@ type Config struct { Traversal string MaxAssetBytes int64 + // AssetSameDomain, when set, localizes only assets whose host shares the + // seed's registrable domain (apple.com covers developer.apple.com and + // www.apple.com but not cdn-apple.com or an unrelated third party). An + // off-domain asset is left pointing at its live URL instead of downloaded. + AssetSameDomain bool + // SkipAssetExts lists asset extensions (".mp4", ".pdf", ".dmg", …) that are + // left on the live web rather than downloaded, so bulk media, installers, and + // archives do not bloat the mirror. The reference keeps its remote URL. + SkipAssetExts map[string]bool + Timeout time.Duration // per HTTP request Settle time.Duration // network-idle quiet period RenderTimeout time.Duration // hard cap per page render @@ -70,25 +80,52 @@ type Config struct { const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" +// DefaultSkipAssetExts returns the asset extensions kage leaves on the live web +// by default: bulk media, installers, and archives that rarely matter for +// reading a site offline but dominate its download size (a docs site's WWDC +// videos, .dmg/.pkg installers, and PDF manuals can be most of the bytes). +// Page-rendering assets (images, fonts, CSS) are deliberately absent, so the +// offline pages still look right. +func DefaultSkipAssetExts() map[string]bool { + exts := []string{ + // Video and audio. + ".mp4", ".m4v", ".mov", ".avi", ".mkv", ".webm", ".flv", ".wmv", + ".m3u8", ".ts", ".mp3", ".wav", ".flac", ".aac", ".ogg", ".oga", + // Installers and disk images. + ".dmg", ".pkg", ".exe", ".msi", ".deb", ".rpm", ".appimage", ".iso", + // Archives. + ".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".7z", ".rar", + // Documents that download rather than render. + ".pdf", + } + m := make(map[string]bool, len(exts)) + for _, e := range exts { + m[e] = true + } + return m +} + // DefaultConfig returns the baseline configuration. func DefaultConfig() Config { return Config{ - OutDir: DefaultOutDir(), - Reserved: urlx.DefaultReserved, - Workers: 4, - AssetWorkers: 8, - BrowserPages: 4, - MaxAssetBytes: 25 << 20, - Traversal: "bfs", - Timeout: 30 * time.Second, - Settle: 1500 * time.Millisecond, - RenderTimeout: 30 * time.Second, - UserAgent: DefaultUserAgent, - RespectRobots: true, - FollowSitemap: true, - Headless: true, - Resume: true, - Persist: true, + OutDir: DefaultOutDir(), + Reserved: urlx.DefaultReserved, + Workers: 4, + AssetWorkers: 8, + BrowserPages: 4, + MaxAssetBytes: 25 << 20, + AssetSameDomain: true, + SkipAssetExts: DefaultSkipAssetExts(), + Traversal: "bfs", + Timeout: 30 * time.Second, + Settle: 1500 * time.Millisecond, + RenderTimeout: 30 * time.Second, + UserAgent: DefaultUserAgent, + RespectRobots: true, + FollowSitemap: true, + Headless: true, + Resume: true, + Persist: true, } } diff --git a/clone/stats.go b/clone/stats.go index 93aff68..635ed7d 100644 --- a/clone/stats.go +++ b/clone/stats.go @@ -12,13 +12,14 @@ const maxRecordedFailures = 100 // stats are the live counters of a run, read by the CLI's progress ticker. type stats struct { - pages atomic.Int64 // page documents written (one per output file) - pagePaths atomic.Int64 // distinct URL paths among those, ignoring query - pagesLinked atomic.Int64 // pages stored as a hard link to identical content - assets atomic.Int64 - pageErrors atomic.Int64 - assetErrors atomic.Int64 - skipped atomic.Int64 // robots-disallowed or out of budget + pages atomic.Int64 // page documents written (one per output file) + pagePaths atomic.Int64 // distinct URL paths among those, ignoring query + pagesLinked atomic.Int64 // pages stored as a hard link to identical content + assets atomic.Int64 + assetSkipped atomic.Int64 // assets left on the live web (over the size cap) + pageErrors atomic.Int64 + assetErrors atomic.Int64 + skipped atomic.Int64 // robots-disallowed or out of budget muPaths sync.Mutex seenPath map[string]struct{} @@ -78,24 +79,26 @@ func (s *stats) recordedFailures() []Failure { // distinct URL paths those represent once query strings are ignored. The // difference, Pages-PagePaths, is the number of query-string variants. type Progress struct { - Pages int64 - PagePaths int64 - PagesLinked int64 - Assets int64 - PageErrors int64 - AssetErrors int64 - Skipped int64 + Pages int64 + PagePaths int64 + PagesLinked int64 + Assets int64 + AssetSkipped int64 + PageErrors int64 + AssetErrors int64 + Skipped int64 } func (s *stats) snapshot() Progress { return Progress{ - Pages: s.pages.Load(), - PagePaths: s.pagePaths.Load(), - PagesLinked: s.pagesLinked.Load(), - Assets: s.assets.Load(), - PageErrors: s.pageErrors.Load(), - AssetErrors: s.assetErrors.Load(), - Skipped: s.skipped.Load(), + Pages: s.pages.Load(), + PagePaths: s.pagePaths.Load(), + PagesLinked: s.pagesLinked.Load(), + Assets: s.assets.Load(), + AssetSkipped: s.assetSkipped.Load(), + PageErrors: s.pageErrors.Load(), + AssetErrors: s.assetErrors.Load(), + Skipped: s.skipped.Load(), } } diff --git a/clone/wantasset_test.go b/clone/wantasset_test.go new file mode 100644 index 0000000..fc91d37 --- /dev/null +++ b/clone/wantasset_test.go @@ -0,0 +1,70 @@ +package clone + +import ( + "testing" + + "github.com/tamnd/kage/urlx" +) + +// TestWantAsset checks the two URL-only skip rules a page worker applies before +// downloading an asset: off-domain hosts and bulk-media extensions are left on +// the live web, while the seed's own images, fonts, and stylesheets localize. +func TestWantAsset(t *testing.T) { + seed, _ := urlx.ParseSeed("https://developer.apple.com/") + c := New(seed, DefaultConfig(), nil) + + cases := []struct { + u string + want bool + }{ + // Same registrable domain: localize. + {"https://developer.apple.com/css/main.css", true}, + {"https://www.apple.com/img/logo.png", true}, + {"https://images.apple.com/fonts/sf.woff2", true}, + // Bulk media, installers, archives, and PDFs: leave remote. + {"https://developer.apple.com/videos/wwdc.mp4", false}, + {"https://developer.apple.com/downloads/Xcode.dmg", false}, + {"https://developer.apple.com/bundle.zip", false}, + {"https://developer.apple.com/guide.pdf", false}, + {"https://developer.apple.com/clip.MP4", false}, // case-insensitive + // Off-domain hosts: leave remote even for a normal image. + {"https://cdn-apple.com/img/x.png", false}, + {"https://ec.europa.eu/banner.png", false}, + {"https://mmbiz.qpic.cn/x.jpg", false}, + } + for _, tc := range cases { + u := mustURL(t, tc.u) + if got := c.wantAsset(u); got != tc.want { + t.Errorf("wantAsset(%q) = %v, want %v", tc.u, got, tc.want) + } + } +} + +// TestWantAssetAllHosts checks that turning the domain scope off localizes a +// third-party asset, while the media extension rule still applies. +func TestWantAssetAllHosts(t *testing.T) { + seed, _ := urlx.ParseSeed("https://developer.apple.com/") + cfg := DefaultConfig() + cfg.AssetSameDomain = false + c := New(seed, cfg, nil) + + if !c.wantAsset(mustURL(t, "https://cdn-apple.com/img/x.png")) { + t.Error("with AssetSameDomain off, an off-domain image should localize") + } + if c.wantAsset(mustURL(t, "https://cdn-apple.com/video/x.mp4")) { + t.Error("a media file should still be skipped regardless of host scope") + } +} + +// TestWantAssetKeepMedia checks that an empty skip set (the --keep-media case) +// localizes media too. +func TestWantAssetKeepMedia(t *testing.T) { + seed, _ := urlx.ParseSeed("https://developer.apple.com/") + cfg := DefaultConfig() + cfg.SkipAssetExts = map[string]bool{} + c := New(seed, cfg, nil) + + if !c.wantAsset(mustURL(t, "https://developer.apple.com/videos/wwdc.mp4")) { + t.Error("with an empty skip set, an on-domain video should localize") + } +} diff --git a/urlx/urlx.go b/urlx/urlx.go index f0cc1f4..de618c4 100644 --- a/urlx/urlx.go +++ b/urlx/urlx.go @@ -19,6 +19,8 @@ import ( "net/url" "path" "strings" + + "golang.org/x/net/publicsuffix" ) // Kind distinguishes a crawlable page from a downloadable asset; the two map to @@ -218,6 +220,34 @@ func SameSite(seed, u *url.URL, allowSub bool) bool { return false } +// SameRegistrableDomain reports whether u shares the seed's registrable domain +// (its eTLD+1). It is looser than SameSite: developer.apple.com, www.apple.com, +// and images.apple.com all fold to apple.com and count as same-domain, while a +// separate brand like cdn-apple.com or an unrelated third party like +// ec.europa.eu does not. It is how kage decides whether an asset host is "the +// site's own" without listing every subdomain a CDN might use. When either host +// has no registrable domain (an IP, an oddball TLD), it falls back to an exact +// host match so the decision stays conservative. +func SameRegistrableDomain(seed, u *url.URL) bool { + sh, uh := seed.Hostname(), u.Hostname() + if sh == uh { + return true + } + sd, err1 := publicsuffix.EffectiveTLDPlusOne(sh) + ud, err2 := publicsuffix.EffectiveTLDPlusOne(uh) + if err1 != nil || err2 != nil { + return false + } + return sd == ud +} + +// Ext returns the lowercased file extension of a URL's last path segment, +// including the leading dot (".pdf", ".mp4"), or "" when there is none. It +// ignores the query string, so "/a/clip.mp4?v=2" reports ".mp4". +func Ext(u *url.URL) string { + return strings.ToLower(path.Ext(lastSegment(u.Path))) +} + // InScope reports whether u should be crawled as a page given the seed and cfg. func InScope(seed, u *url.URL, cfg ScopeConfig) bool { if !SameSite(seed, u, cfg.IncludeSubdomains) { diff --git a/urlx/urlx_test.go b/urlx/urlx_test.go index cc78a38..265f0b0 100644 --- a/urlx/urlx_test.go +++ b/urlx/urlx_test.go @@ -116,6 +116,40 @@ func TestInScope(t *testing.T) { } } +func TestSameRegistrableDomain(t *testing.T) { + seed := mustParse(t, "https://developer.apple.com/") + cases := map[string]bool{ + "https://developer.apple.com/x": true, + "https://www.apple.com/x": true, + "https://images.apple.com/x": true, + "https://apple.com/x": true, + "https://cdn-apple.com/x": false, + "https://ec.europa.eu/x": false, + "https://mmbiz.qpic.cn/x": false, + } + for u, want := range cases { + if got := SameRegistrableDomain(seed, mustParse(t, u)); got != want { + t.Errorf("SameRegistrableDomain(%q) = %v, want %v", u, got, want) + } + } +} + +func TestExt(t *testing.T) { + cases := map[string]string{ + "https://ex.com/a/clip.mp4": ".mp4", + "https://ex.com/a/clip.MP4?v=2": ".mp4", + "https://ex.com/style.css": ".css", + "https://ex.com/docs/": "", + "https://ex.com/docs": "", + "https://ex.com/Xcode_15.0.dmg": ".dmg", + } + for u, want := range cases { + if got := Ext(mustParse(t, u)); got != want { + t.Errorf("Ext(%q) = %q, want %q", u, got, want) + } + } +} + func TestLikelyPage(t *testing.T) { cases := map[string]bool{ "https://ex.com/docs": true,