Duc-Tam Nguyen
bf850c6b93
An extensionless link is queued as a page, so the page worker navigated to it in headless Chrome. When such a link served a binary, a zip or a CSV, Chrome saved the file to the user's Downloads folder, a surprise side effect of a clone (issue #32). Deny Chrome-initiated downloads browser-wide, since kage fetches every asset through its own downloader and never needs the browser to write a file. Then watch the main document's response, and when it is not HTML, return a typed ErrNotHTML so the page worker reroutes the URL to the asset downloader, where the existing size and media policy decides whether to localise it or leave it on the live web. Verified against the two URLs from the issue, a zip and a CSV: both land under the mirror's reserved tree and nothing is written to Downloads.
375 lines
11 KiB
Go
375 lines
11 KiB
Go
package clone
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/tamnd/kage/browser"
|
|
"github.com/tamnd/kage/urlx"
|
|
)
|
|
|
|
// testSite is a tiny two-page site with a stylesheet, an image, an inline
|
|
// script, an onclick handler, and a javascript: link, so a full clone exercises
|
|
// rendering, asset localisation, and JavaScript stripping at once.
|
|
func testSite(t *testing.T) *httptest.Server {
|
|
t.Helper()
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
_, _ = w.Write([]byte(`<!doctype html><html><head>
|
|
<link rel="stylesheet" href="/site.css">
|
|
<script src="/app.js"></script>
|
|
</head><body>
|
|
<h1>Home</h1>
|
|
<img src="/logo.png" alt="logo">
|
|
<a href="/about">About</a>
|
|
<a href="javascript:void(0)" onclick="boom()">Danger</a>
|
|
<script>console.log("inline")</script>
|
|
</body></html>`))
|
|
})
|
|
mux.HandleFunc("/about", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
_, _ = w.Write([]byte(`<!doctype html><html><head>
|
|
<link rel="stylesheet" href="/site.css">
|
|
</head><body><h1>About</h1><a href="/">Home</a></body></html>`))
|
|
})
|
|
mux.HandleFunc("/site.css", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/css")
|
|
_, _ = w.Write([]byte(`body{background:url("/bg.png")} h1{color:red}`))
|
|
})
|
|
mux.HandleFunc("/app.js", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/javascript")
|
|
_, _ = w.Write([]byte(`document.body.dataset.ran = "1";`))
|
|
})
|
|
mux.HandleFunc("/logo.png", servePNG)
|
|
mux.HandleFunc("/bg.png", servePNG)
|
|
mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = w.Write([]byte("User-agent: *\nAllow: /\n"))
|
|
})
|
|
return httptest.NewServer(mux)
|
|
}
|
|
|
|
// a 1x1 transparent PNG.
|
|
var pngBytes = []byte{
|
|
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
|
|
0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
|
|
0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89, 0x00, 0x00, 0x00,
|
|
0x0a, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0x00, 0x01, 0x00, 0x00,
|
|
0x05, 0x00, 0x01, 0x0d, 0x0a, 0x2d, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x49,
|
|
0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
|
|
}
|
|
|
|
func servePNG(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "image/png")
|
|
_, _ = w.Write(pngBytes)
|
|
}
|
|
|
|
func TestCloneEndToEnd(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("clone end-to-end drives Chrome; skipped under -short")
|
|
}
|
|
if _, ok := browser.LookChrome(); !ok {
|
|
t.Skip("no Chrome/Chromium found; skipping clone end-to-end")
|
|
}
|
|
|
|
srv := testSite(t)
|
|
defer srv.Close()
|
|
|
|
seed, err := urlx.ParseSeed(srv.URL)
|
|
if err != nil {
|
|
t.Fatalf("parse seed: %v", err)
|
|
}
|
|
|
|
out := t.TempDir()
|
|
cfg := DefaultConfig()
|
|
cfg.OutDir = out
|
|
cfg.Settle = 300 * time.Millisecond
|
|
cfg.RenderTimeout = 20 * time.Second
|
|
cfg.Timeout = 10 * time.Second
|
|
|
|
c := New(seed, cfg, t.Logf)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
|
|
res, err := c.Run(ctx)
|
|
if err != nil {
|
|
t.Fatalf("run: %v", err)
|
|
}
|
|
if res.Pages < 2 {
|
|
t.Fatalf("expected at least 2 pages, got %d", res.Pages)
|
|
}
|
|
|
|
root := res.OutDir
|
|
indexPath := filepath.Join(root, "index.html")
|
|
body := readFile(t, indexPath)
|
|
|
|
// JavaScript is gone: no <script>, no onclick, no javascript: URL.
|
|
if strings.Contains(strings.ToLower(body), "<script") {
|
|
t.Error("index.html still contains a <script> tag")
|
|
}
|
|
if strings.Contains(strings.ToLower(body), "onclick") {
|
|
t.Error("index.html still contains an onclick handler")
|
|
}
|
|
if strings.Contains(strings.ToLower(body), "javascript:") {
|
|
t.Error("index.html still contains a javascript: URL")
|
|
}
|
|
|
|
// Layout is preserved: the stylesheet link survives and points local.
|
|
if !strings.Contains(body, "stylesheet") {
|
|
t.Error("index.html lost its stylesheet link")
|
|
}
|
|
if strings.Contains(body, srv.URL+"/site.css") {
|
|
t.Error("stylesheet still points at the live origin")
|
|
}
|
|
|
|
// The about page and the localised assets exist on disk.
|
|
if !fileExists(filepath.Join(root, "about", "index.html")) {
|
|
t.Error("about page was not written")
|
|
}
|
|
assetDir := filepath.Join(root, cfg.Reserved)
|
|
if !anyFileUnder(t, assetDir, "site.css") {
|
|
t.Error("site.css was not downloaded")
|
|
}
|
|
if !anyFileUnder(t, assetDir, "logo.png") {
|
|
t.Error("logo.png was not downloaded")
|
|
}
|
|
|
|
// The localised CSS had its url() rewritten away from the origin.
|
|
css := readAnyFile(t, assetDir, "site.css")
|
|
if strings.Contains(css, srv.URL) {
|
|
t.Error("site.css still references the live origin in url()")
|
|
}
|
|
}
|
|
|
|
// TestPageKeyCollapsesDuplicates guards the dedup identity: http vs https, a
|
|
// trailing slash, and "/index.html" vs "/" all name the same output file, so a
|
|
// page is crawled once rather than two-to-four times.
|
|
func TestPageKeyCollapsesDuplicates(t *testing.T) {
|
|
seed, _ := urlx.ParseSeed("https://ex.com")
|
|
c := New(seed, DefaultConfig(), nil)
|
|
|
|
groups := [][]string{
|
|
{"https://ex.com/", "http://ex.com/", "https://ex.com/index.html", "http://ex.com/index.html"},
|
|
{"https://ex.com/docs", "http://ex.com/docs", "https://ex.com/docs/", "https://ex.com/docs/index.html"},
|
|
}
|
|
for _, g := range groups {
|
|
want := c.pageKey(mustURL(t, g[0]))
|
|
for _, raw := range g[1:] {
|
|
if got := c.pageKey(mustURL(t, raw)); got != want {
|
|
t.Errorf("pageKey(%q) = %q, want %q (same page)", raw, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Genuinely different pages must not collapse.
|
|
if c.pageKey(mustURL(t, "https://ex.com/a")) == c.pageKey(mustURL(t, "https://ex.com/b")) {
|
|
t.Error("distinct pages share a key")
|
|
}
|
|
}
|
|
|
|
func mustURL(t *testing.T, raw string) *url.URL {
|
|
t.Helper()
|
|
u, err := url.Parse(raw)
|
|
if err != nil {
|
|
t.Fatalf("parse %q: %v", raw, err)
|
|
}
|
|
return u
|
|
}
|
|
|
|
func TestCloneResumeSkipsVisited(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("resume test drives Chrome; skipped under -short")
|
|
}
|
|
if _, ok := browser.LookChrome(); !ok {
|
|
t.Skip("no Chrome/Chromium found; skipping resume test")
|
|
}
|
|
|
|
srv := testSite(t)
|
|
defer srv.Close()
|
|
seed, _ := urlx.ParseSeed(srv.URL)
|
|
|
|
out := t.TempDir()
|
|
cfg := DefaultConfig()
|
|
cfg.OutDir = out
|
|
cfg.Settle = 300 * time.Millisecond
|
|
|
|
c1 := New(seed, cfg, t.Logf)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
if _, err := c1.Run(ctx); err != nil {
|
|
t.Fatalf("first run: %v", err)
|
|
}
|
|
|
|
// Second run with resume on should find the state and re-render nothing new.
|
|
c2 := New(seed, cfg, t.Logf)
|
|
res2, err := c2.Run(ctx)
|
|
if err != nil {
|
|
t.Fatalf("second run: %v", err)
|
|
}
|
|
if res2.Pages != 0 {
|
|
t.Fatalf("resume should skip all visited pages, but rendered %d", res2.Pages)
|
|
}
|
|
}
|
|
|
|
func TestCloneRefreshReRenders(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("refresh test drives Chrome; skipped under -short")
|
|
}
|
|
if _, ok := browser.LookChrome(); !ok {
|
|
t.Skip("no Chrome/Chromium found; skipping refresh test")
|
|
}
|
|
|
|
srv := testSite(t)
|
|
defer srv.Close()
|
|
seed, _ := urlx.ParseSeed(srv.URL)
|
|
|
|
out := t.TempDir()
|
|
cfg := DefaultConfig()
|
|
cfg.OutDir = out
|
|
cfg.Settle = 300 * time.Millisecond
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
|
defer cancel()
|
|
|
|
if _, err := New(seed, cfg, t.Logf).Run(ctx); err != nil {
|
|
t.Fatalf("first run: %v", err)
|
|
}
|
|
|
|
// Resume keeps everything in place but renders nothing new; refresh, on the
|
|
// same mirror, re-renders every page so changed content is pulled back in.
|
|
cfg.Refresh = true
|
|
res, err := New(seed, cfg, t.Logf).Run(ctx)
|
|
if err != nil {
|
|
t.Fatalf("refresh run: %v", err)
|
|
}
|
|
if res.Pages < 2 {
|
|
t.Fatalf("refresh should re-render every page, got %d", res.Pages)
|
|
}
|
|
if !fileExists(filepath.Join(out, seed.Hostname(), "index.html")) {
|
|
t.Error("refresh removed the mirror instead of overwriting in place")
|
|
}
|
|
}
|
|
|
|
// TestCloneRoutesNonHTMLToAsset guards issue #32: an extensionless link that
|
|
// turns out to be a file (a zip) is classified as a page up front, but once the
|
|
// page worker sees it is not HTML it must be handed to the asset downloader, not
|
|
// saved as a broken page nor downloaded by Chrome to ~/Downloads.
|
|
func TestCloneRoutesNonHTMLToAsset(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("clone test drives Chrome; skipped under -short")
|
|
}
|
|
if _, ok := browser.LookChrome(); !ok {
|
|
t.Skip("no Chrome/Chromium found; skipping clone test")
|
|
}
|
|
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
// The link has no extension, so it is queued as a page; the server then
|
|
// answers it with a zip.
|
|
_, _ = w.Write([]byte(`<!doctype html><html><body>
|
|
<h1>Home</h1><a href="/download">grab the bundle</a></body></html>`))
|
|
})
|
|
mux.HandleFunc("/download", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/zip")
|
|
_, _ = w.Write([]byte("PK\x03\x04 pretend bundle"))
|
|
})
|
|
srv := httptest.NewServer(mux)
|
|
defer srv.Close()
|
|
|
|
seed, err := urlx.ParseSeed(srv.URL)
|
|
if err != nil {
|
|
t.Fatalf("parse seed: %v", err)
|
|
}
|
|
|
|
out := t.TempDir()
|
|
cfg := DefaultConfig()
|
|
cfg.OutDir = out
|
|
cfg.Settle = 300 * time.Millisecond
|
|
cfg.RenderTimeout = 20 * time.Second
|
|
cfg.Timeout = 10 * time.Second
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
|
|
res, err := New(seed, cfg, t.Logf).Run(ctx)
|
|
if err != nil {
|
|
t.Fatalf("run: %v", err)
|
|
}
|
|
|
|
root := res.OutDir
|
|
// The home page is a real page and is written.
|
|
if !fileExists(filepath.Join(root, "index.html")) {
|
|
t.Error("home page was not written")
|
|
}
|
|
// The zip is NOT saved as a page: no download/index.html exists.
|
|
if fileExists(filepath.Join(root, "download", "index.html")) {
|
|
t.Error("non-HTML target was saved as a page")
|
|
}
|
|
// The zip is fetched as an asset under the reserved tree instead.
|
|
if res.Assets < 1 {
|
|
t.Errorf("expected the zip to be fetched as an asset, assets=%d", res.Assets)
|
|
}
|
|
assetDir := filepath.Join(root, cfg.Reserved)
|
|
if !anyFileUnder(t, assetDir, "download") {
|
|
t.Error("the zip was not downloaded into the reserved asset tree")
|
|
}
|
|
if res.PageErrors != 0 {
|
|
t.Errorf("a rerouted non-HTML target must not count as a page error, got %d", res.PageErrors)
|
|
}
|
|
}
|
|
|
|
func readFile(t *testing.T, path string) string {
|
|
t.Helper()
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("read %s: %v", path, err)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func fileExists(path string) bool {
|
|
_, err := os.Stat(path)
|
|
return err == nil
|
|
}
|
|
|
|
func anyFileUnder(t *testing.T, dir, name string) bool {
|
|
t.Helper()
|
|
found := false
|
|
_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
|
if err == nil && !d.IsDir() && strings.HasSuffix(p, name) {
|
|
found = true
|
|
}
|
|
return nil
|
|
})
|
|
return found
|
|
}
|
|
|
|
func readAnyFile(t *testing.T, dir, name string) string {
|
|
t.Helper()
|
|
var out string
|
|
_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
|
if err == nil && !d.IsDir() && strings.HasSuffix(p, name) {
|
|
b, _ := os.ReadFile(p)
|
|
out = string(b)
|
|
}
|
|
return nil
|
|
})
|
|
return out
|
|
}
|