Duc-Tam Nguyen
e6afa91e09
kage renders every page in headless Chrome, snapshots the final DOM, strips all JavaScript, and localises CSS, images, and fonts so a site can be browsed offline as a plain folder of files. The engine is split into small packages: urlx deterministic URL to local-path mapping and scope rules sanitize remove scripts, on* handlers, and javascript: URLs asset rewrite HTML and CSS references, download assets browser headless Chrome pool over the DevTools protocol robots robots.txt matcher clone the orchestrator: a polite resumable breadth-first crawl The cli package wires a cobra and fang command surface with two commands, clone and serve. Every pure package has table tests; the browser and clone packages add Chrome-driven end-to-end tests that skip when no browser is present or under -short. CI runs gofmt, vet, build, race tests, golangci-lint, govulncheck, and a tidy check on Linux and macOS. A goreleaser config fans one tag out to archives, deb/rpm/apk, a Chromium-bundled GHCR image, and the package managers. A tago docs site builds to Pages and Cloudflare.
127 lines
3.8 KiB
Go
127 lines
3.8 KiB
Go
package sanitize
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
const page = `<!doctype html>
|
|
<html><head>
|
|
<meta charset="utf-8">
|
|
<meta http-equiv="refresh" content="5;url=https://ex.com/next">
|
|
<title>Hi</title>
|
|
<link rel="stylesheet" href="/css/main.css">
|
|
<link rel="preconnect" href="https://cdn.io">
|
|
<link rel="modulepreload" href="/app.js">
|
|
<link rel="preload" as="script" href="/runtime.js">
|
|
<style>.a{color:red}</style>
|
|
<script src="/vendor.js"></script>
|
|
<script>window.x=1</script>
|
|
</head>
|
|
<body onload="boot()">
|
|
<h1 onclick="go()">Title</h1>
|
|
<a href="javascript:open()">js link</a>
|
|
<a href="/real">real link</a>
|
|
<img src="/logo.png" onerror="fail()">
|
|
<form action="/submit"><input name="q"></form>
|
|
<noscript><p>need js</p></noscript>
|
|
<p style="background:url(/bg.png)">styled</p>
|
|
</body></html>`
|
|
|
|
func TestStripRemovesAllJS(t *testing.T) {
|
|
out, rep, err := Strip([]byte(page), Options{Banner: "cloned by kage"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s := string(out)
|
|
|
|
if strings.Contains(s, "<script") {
|
|
t.Error("a <script> survived")
|
|
}
|
|
if strings.Contains(s, "onload") || strings.Contains(s, "onclick") || strings.Contains(s, "onerror") {
|
|
t.Error("an on* handler survived")
|
|
}
|
|
if strings.Contains(strings.ToLower(s), "javascript:") {
|
|
t.Error("a javascript: URL survived")
|
|
}
|
|
if strings.Contains(s, "modulepreload") || strings.Contains(s, "preconnect") {
|
|
t.Error("a dead resource hint survived")
|
|
}
|
|
if strings.Contains(s, "http-equiv") {
|
|
t.Error("a meta refresh survived")
|
|
}
|
|
|
|
// Layout-bearing markup must survive untouched.
|
|
for _, want := range []string{
|
|
`rel="stylesheet"`, `href="/css/main.css"`,
|
|
`<style>`, `color:red`,
|
|
`src="/logo.png"`, `<form action="/submit">`,
|
|
`background:url(/bg.png)`, `href="/real"`,
|
|
"<!-- cloned by kage -->",
|
|
} {
|
|
if !strings.Contains(s, want) {
|
|
t.Errorf("expected %q to survive, output:\n%s", want, s)
|
|
}
|
|
}
|
|
|
|
// The js link keeps an anchor but points nowhere dangerous.
|
|
if !strings.Contains(s, `href="#"`) {
|
|
t.Error("javascript: link should be neutralized to href=#")
|
|
}
|
|
|
|
if rep.ScriptsRemoved != 2 {
|
|
t.Errorf("ScriptsRemoved = %d, want 2", rep.ScriptsRemoved)
|
|
}
|
|
if rep.HandlersRemoved != 3 {
|
|
t.Errorf("HandlersRemoved = %d, want 3", rep.HandlersRemoved)
|
|
}
|
|
if rep.JSURLsNeutralized != 1 {
|
|
t.Errorf("JSURLsNeutralized = %d, want 1", rep.JSURLsNeutralized)
|
|
}
|
|
if rep.MetaRefreshRemoved != 1 {
|
|
t.Errorf("MetaRefreshRemoved = %d, want 1", rep.MetaRefreshRemoved)
|
|
}
|
|
if rep.DeadLinksRemoved != 3 {
|
|
t.Errorf("DeadLinksRemoved = %d, want 3", rep.DeadLinksRemoved)
|
|
}
|
|
if rep.NoscriptRemoved != 1 {
|
|
t.Errorf("NoscriptRemoved = %d, want 1", rep.NoscriptRemoved)
|
|
}
|
|
}
|
|
|
|
func TestKeepNoscriptUnwraps(t *testing.T) {
|
|
in := `<html><body><noscript><p>fallback text</p></noscript></body></html>`
|
|
out, rep, err := Strip([]byte(in), Options{KeepNoscript: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s := string(out)
|
|
if strings.Contains(s, "<noscript") {
|
|
t.Error("noscript wrapper should be gone")
|
|
}
|
|
if !strings.Contains(s, "fallback text") {
|
|
t.Errorf("unwrapped content missing: %s", s)
|
|
}
|
|
if rep.NoscriptUnwrapped != 1 {
|
|
t.Errorf("NoscriptUnwrapped = %d, want 1", rep.NoscriptUnwrapped)
|
|
}
|
|
}
|
|
|
|
func TestKeepMetaRefreshPlain(t *testing.T) {
|
|
in := `<html><head><meta http-equiv="refresh" content="5;url=/next"></head><body></body></html>`
|
|
out, _, err := Strip([]byte(in), Options{KeepMetaRefresh: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(out), "http-equiv") {
|
|
t.Error("plain meta refresh should be kept when KeepMetaRefresh is set")
|
|
}
|
|
|
|
// A JS-target refresh is removed even when KeepMetaRefresh is set.
|
|
js := `<html><head><meta http-equiv="refresh" content="0;url=javascript:alert(1)"></head><body></body></html>`
|
|
out2, _, _ := Strip([]byte(js), Options{KeepMetaRefresh: true})
|
|
if strings.Contains(strings.ToLower(string(out2)), "javascript:") {
|
|
t.Error("JS-target meta refresh must be removed regardless")
|
|
}
|
|
}
|