Duc-Tam Nguyen
249761f29f
A developer.apple.com crawl came back at 19 GB, 18 of it assets, most of that the site's own videos, .dmg/.pkg installers, and PDF manuals pulled from a couple dozen hosts including unrelated third parties. None of it helps read the docs offline. This changes what kage localizes by default. Bulk media, installers, archives, and PDFs are left pointing at their live URL instead of downloaded. The decision is made from the URL alone, so the rewritten HTML simply keeps the remote link. --keep-media restores the old behavior and --skip-ext adds more extensions to leave remote. Assets are localized only from the seed's registrable domain by default, so www.apple.com and images.apple.com still come along but a separate brand domain or an off-topic third party does not. --all-asset-hosts goes back to downloading from any host. The size cap was also truncating instead of skipping: it wrapped the body in a LimitReader, so an over-cap file was saved as exactly the first N MB of itself, a corrupt fragment that would never play or run. On the apple crawl that was around a gigabyte of half-downloaded WWDC videos. kage now checks the response size and leaves an over-cap asset out of the mirror.
71 lines
2.4 KiB
Go
71 lines
2.4 KiB
Go
package clone
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/tamnd/kage/urlx"
|
|
)
|
|
|
|
// TestWantAsset checks the two URL-only skip rules a page worker applies before
|
|
// downloading an asset: off-domain hosts and bulk-media extensions are left on
|
|
// the live web, while the seed's own images, fonts, and stylesheets localize.
|
|
func TestWantAsset(t *testing.T) {
|
|
seed, _ := urlx.ParseSeed("https://developer.apple.com/")
|
|
c := New(seed, DefaultConfig(), nil)
|
|
|
|
cases := []struct {
|
|
u string
|
|
want bool
|
|
}{
|
|
// Same registrable domain: localize.
|
|
{"https://developer.apple.com/css/main.css", true},
|
|
{"https://www.apple.com/img/logo.png", true},
|
|
{"https://images.apple.com/fonts/sf.woff2", true},
|
|
// Bulk media, installers, archives, and PDFs: leave remote.
|
|
{"https://developer.apple.com/videos/wwdc.mp4", false},
|
|
{"https://developer.apple.com/downloads/Xcode.dmg", false},
|
|
{"https://developer.apple.com/bundle.zip", false},
|
|
{"https://developer.apple.com/guide.pdf", false},
|
|
{"https://developer.apple.com/clip.MP4", false}, // case-insensitive
|
|
// Off-domain hosts: leave remote even for a normal image.
|
|
{"https://cdn-apple.com/img/x.png", false},
|
|
{"https://ec.europa.eu/banner.png", false},
|
|
{"https://mmbiz.qpic.cn/x.jpg", false},
|
|
}
|
|
for _, tc := range cases {
|
|
u := mustURL(t, tc.u)
|
|
if got := c.wantAsset(u); got != tc.want {
|
|
t.Errorf("wantAsset(%q) = %v, want %v", tc.u, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestWantAssetAllHosts checks that turning the domain scope off localizes a
|
|
// third-party asset, while the media extension rule still applies.
|
|
func TestWantAssetAllHosts(t *testing.T) {
|
|
seed, _ := urlx.ParseSeed("https://developer.apple.com/")
|
|
cfg := DefaultConfig()
|
|
cfg.AssetSameDomain = false
|
|
c := New(seed, cfg, nil)
|
|
|
|
if !c.wantAsset(mustURL(t, "https://cdn-apple.com/img/x.png")) {
|
|
t.Error("with AssetSameDomain off, an off-domain image should localize")
|
|
}
|
|
if c.wantAsset(mustURL(t, "https://cdn-apple.com/video/x.mp4")) {
|
|
t.Error("a media file should still be skipped regardless of host scope")
|
|
}
|
|
}
|
|
|
|
// TestWantAssetKeepMedia checks that an empty skip set (the --keep-media case)
|
|
// localizes media too.
|
|
func TestWantAssetKeepMedia(t *testing.T) {
|
|
seed, _ := urlx.ParseSeed("https://developer.apple.com/")
|
|
cfg := DefaultConfig()
|
|
cfg.SkipAssetExts = map[string]bool{}
|
|
c := New(seed, cfg, nil)
|
|
|
|
if !c.wantAsset(mustURL(t, "https://developer.apple.com/videos/wwdc.mp4")) {
|
|
t.Error("with an empty skip set, an on-domain video should localize")
|
|
}
|
|
}
|