Duc-Tam Nguyen
e6afa91e09
kage renders every page in headless Chrome, snapshots the final DOM, strips all JavaScript, and localises CSS, images, and fonts so a site can be browsed offline as a plain folder of files. The engine is split into small packages: urlx deterministic URL to local-path mapping and scope rules sanitize remove scripts, on* handlers, and javascript: URLs asset rewrite HTML and CSS references, download assets browser headless Chrome pool over the DevTools protocol robots robots.txt matcher clone the orchestrator: a polite resumable breadth-first crawl The cli package wires a cobra and fang command surface with two commands, clone and serve. Every pure package has table tests; the browser and clone packages add Chrome-driven end-to-end tests that skip when no browser is present or under -short. CI runs gofmt, vet, build, race tests, golangci-lint, govulncheck, and a tidy check on Linux and macOS. A goreleaser config fans one tag out to archives, deb/rpm/apk, a Chromium-bundled GHCR image, and the package managers. A tago docs site builds to Pages and Cloudflare.
117 lines
4.0 KiB
Go
117 lines
4.0 KiB
Go
package asset
|
|
|
|
import (
|
|
"bytes"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/tamnd/kage/urlx"
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
// testSink builds a RefSink that records every URL it sees and rewrites it to a
|
|
// local relative path from fromDir, mirroring what the cloner does.
|
|
func testSink(seedHost, fromDir string, seen *[]string) RefSink {
|
|
return func(u *url.URL, kind urlx.Kind) string {
|
|
*seen = append(*seen, u.String())
|
|
if !urlx.SameSite(&url.URL{Host: seedHost}, u, true) && kind == urlx.Page {
|
|
return u.String() // external page links stay absolute
|
|
}
|
|
local := urlx.LocalPath(seedHost, u, kind, "")
|
|
return urlx.Rel(fromDir, local)
|
|
}
|
|
}
|
|
|
|
func TestRewriteCSS(t *testing.T) {
|
|
base, _ := url.Parse("https://ex.com/css/main.css")
|
|
in := `@import "base.css";
|
|
@import url('https://cdn.io/reset.css');
|
|
body { background: url(../img/bg.png); }
|
|
.x { background: url("https://cdn.io/sprite.svg"); }
|
|
.y { content: url(data:image/gif;base64,AAAA); }`
|
|
var seen []string
|
|
// CSS lives at _kage/ex.com/css/main.css → its dir.
|
|
fromDir := "_kage/ex.com/css"
|
|
out := string(RewriteCSS([]byte(in), base, testSink("ex.com", fromDir, &seen)))
|
|
|
|
if !strings.Contains(out, `@import "base.css"`) {
|
|
t.Errorf("local @import not rewritten relative: %s", out)
|
|
}
|
|
if !strings.Contains(out, `url("../img/bg.png")`) {
|
|
t.Errorf("local url() not rewritten relative: %s", out)
|
|
}
|
|
// Cross-origin asset goes under _kage/cdn.io/...; from _kage/ex.com/css that
|
|
// is ../../cdn.io/...
|
|
if !strings.Contains(out, `url("../../cdn.io/reset.css")`) {
|
|
t.Errorf("cross-origin @import not localised: %s", out)
|
|
}
|
|
if !strings.Contains(out, `url("../../cdn.io/sprite.svg")`) {
|
|
t.Errorf("cross-origin url() not localised: %s", out)
|
|
}
|
|
// data: URL must be left exactly as-is.
|
|
if !strings.Contains(out, "url(data:image/gif;base64,AAAA)") {
|
|
t.Errorf("data: URL was altered: %s", out)
|
|
}
|
|
|
|
sort.Strings(seen)
|
|
want := []string{
|
|
"https://cdn.io/reset.css",
|
|
"https://cdn.io/sprite.svg",
|
|
"https://ex.com/css/base.css",
|
|
"https://ex.com/img/bg.png",
|
|
}
|
|
if strings.Join(seen, ",") != strings.Join(want, ",") {
|
|
t.Errorf("sink saw %v, want %v", seen, want)
|
|
}
|
|
}
|
|
|
|
const htmlDoc = `<!doctype html><html><head>
|
|
<link rel="stylesheet" href="/css/main.css">
|
|
<link rel="icon" href="/favicon.ico">
|
|
<link rel="canonical" href="https://ex.com/canon">
|
|
</head><body>
|
|
<a href="/docs/intro">internal</a>
|
|
<a href="https://other.com/x">external</a>
|
|
<a href="/files/report.pdf">a pdf</a>
|
|
<img src="/img/logo.png" srcset="/img/logo.png 1x, /img/logo@2x.png 2x">
|
|
<p style="background:url(/img/bg.png)">hi</p>
|
|
<style>.a{background:url(/img/tile.png)}</style>
|
|
</body></html>`
|
|
|
|
func TestRewriteHTML(t *testing.T) {
|
|
base, _ := url.Parse("https://ex.com/")
|
|
root, err := html.Parse(strings.NewReader(htmlDoc))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var seen []string
|
|
// The page is the root index.html, so fromDir is "".
|
|
RewriteHTML(root, base, testSink("ex.com", "", &seen))
|
|
|
|
var buf bytes.Buffer
|
|
if err := html.Render(&buf, root); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out := buf.String()
|
|
|
|
checks := map[string]bool{
|
|
`href="_kage/ex.com/css/main.css"`: true, // stylesheet localised
|
|
`href="_kage/ex.com/favicon.ico"`: true, // icon localised
|
|
`href="https://ex.com/canon"`: true, // canonical left alone
|
|
`href="docs/intro/index.html"`: true, // internal page → local
|
|
`href="https://other.com/x"`: true, // external page stays absolute
|
|
`href="_kage/ex.com/files/report.pdf"`: true, // pdf link treated as asset
|
|
`src="_kage/ex.com/img/logo.png"`: true,
|
|
`_kage/ex.com/img/logo@2x.png 2x`: true, // srcset candidate rewritten
|
|
`background:url("_kage/ex.com/img/bg.png")`: true, // inline style (attr-escaped quotes)
|
|
`background:url("_kage/ex.com/img/tile.png")`: true, // <style> text
|
|
}
|
|
for frag, want := range checks {
|
|
if strings.Contains(out, frag) != want {
|
|
t.Errorf("fragment %q present=%v, want %v\n--- output ---\n%s", frag, !want, want, out)
|
|
}
|
|
}
|
|
}
|