Add the clone engine, CLI, tests, CI, and docs
kage renders every page in headless Chrome, snapshots the final DOM, strips all JavaScript, and localises CSS, images, and fonts so a site can be browsed offline as a plain folder of files. The engine is split into small packages: urlx deterministic URL to local-path mapping and scope rules sanitize remove scripts, on* handlers, and javascript: URLs asset rewrite HTML and CSS references, download assets browser headless Chrome pool over the DevTools protocol robots robots.txt matcher clone the orchestrator: a polite resumable breadth-first crawl The cli package wires a cobra and fang command surface with two commands, clone and serve. Every pure package has table tests; the browser and clone packages add Chrome-driven end-to-end tests that skip when no browser is present or under -short. CI runs gofmt, vet, build, race tests, golangci-lint, govulncheck, and a tidy check on Linux and macOS. A goreleaser config fans one tag out to archives, deb/rpm/apk, a Chromium-bundled GHCR image, and the package managers. A tago docs site builds to Pages and Cloudflare.
This commit is contained in:
Duc-Tam Nguyen
@@ -0,0 +1,218 @@
|
||||
// Package robots is a small, pure robots.txt parser and matcher. kage obeys it
|
||||
// by default so a clone stays polite; --no-robots bypasses the matcher and puts
|
||||
// the consequences on the user.
|
||||
package robots
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// rule is one Allow or Disallow directive.
|
||||
type rule struct {
|
||||
pattern string
|
||||
allow bool
|
||||
}
|
||||
|
||||
// Matcher answers whether a path may be crawled for one user-agent group, and
|
||||
// carries the sitemaps and crawl-delay declared in the file.
|
||||
type Matcher struct {
|
||||
rules []rule
|
||||
Sitemaps []string
|
||||
CrawlDelay time.Duration
|
||||
}
|
||||
|
||||
// AllowAll returns a Matcher that permits every path (used when robots is
|
||||
// disabled or the file is absent).
|
||||
func AllowAll() *Matcher { return &Matcher{} }
|
||||
|
||||
// group accumulates the rules attached to a set of user-agent lines.
|
||||
type group struct {
|
||||
agents []string
|
||||
rules []rule
|
||||
delay time.Duration
|
||||
}
|
||||
|
||||
// Parse reads robots.txt content and returns a Matcher for the given agent
|
||||
// token (e.g. "kage"). The most specific matching group is used, falling back to
|
||||
// the "*" group. Sitemaps are collected globally.
|
||||
func Parse(data string, agent string) *Matcher {
|
||||
agent = strings.ToLower(agent)
|
||||
var groups []*group
|
||||
var cur *group
|
||||
var sitemaps []string
|
||||
startedRules := false
|
||||
|
||||
for line := range strings.SplitSeq(data, "\n") {
|
||||
line = stripComment(line)
|
||||
key, val, ok := splitField(line)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch strings.ToLower(key) {
|
||||
case "user-agent":
|
||||
// A user-agent line after rules starts a fresh group.
|
||||
if cur == nil || startedRules {
|
||||
cur = &group{}
|
||||
groups = append(groups, cur)
|
||||
startedRules = false
|
||||
}
|
||||
cur.agents = append(cur.agents, strings.ToLower(val))
|
||||
case "allow":
|
||||
if cur != nil {
|
||||
cur.rules = append(cur.rules, rule{pattern: val, allow: true})
|
||||
startedRules = true
|
||||
}
|
||||
case "disallow":
|
||||
if cur != nil {
|
||||
cur.rules = append(cur.rules, rule{pattern: val, allow: false})
|
||||
startedRules = true
|
||||
}
|
||||
case "crawl-delay":
|
||||
if cur != nil {
|
||||
if d := parseDelay(val); d > 0 {
|
||||
cur.delay = d
|
||||
}
|
||||
startedRules = true
|
||||
}
|
||||
case "sitemap":
|
||||
if val != "" {
|
||||
sitemaps = append(sitemaps, val)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m := &Matcher{Sitemaps: sitemaps}
|
||||
if g := selectGroup(groups, agent); g != nil {
|
||||
m.rules = g.rules
|
||||
m.CrawlDelay = g.delay
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// selectGroup picks the group whose agent best matches: an exact/substring
|
||||
// match on the agent token beats the wildcard "*" group.
|
||||
func selectGroup(groups []*group, agent string) *group {
|
||||
var star, specific *group
|
||||
bestLen := -1
|
||||
for _, g := range groups {
|
||||
for _, a := range g.agents {
|
||||
if a == "*" {
|
||||
star = g
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(agent, a) || strings.Contains(agent, a) {
|
||||
if len(a) > bestLen {
|
||||
bestLen = len(a)
|
||||
specific = g
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if specific != nil {
|
||||
return specific
|
||||
}
|
||||
return star
|
||||
}
|
||||
|
||||
// Allowed reports whether path may be crawled. The longest matching rule wins;
|
||||
// on a tie, Allow beats Disallow. An empty Disallow means "allow everything".
|
||||
func (m *Matcher) Allowed(path string) bool {
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
bestLen := -1
|
||||
bestAllow := true
|
||||
for _, r := range m.rules {
|
||||
if r.pattern == "" {
|
||||
// Empty Disallow = allow all; empty Allow is a no-op.
|
||||
continue
|
||||
}
|
||||
if matchPattern(r.pattern, path) {
|
||||
pl := len(r.pattern)
|
||||
if pl > bestLen || (pl == bestLen && r.allow) {
|
||||
bestLen = pl
|
||||
bestAllow = r.allow
|
||||
}
|
||||
}
|
||||
}
|
||||
if bestLen < 0 {
|
||||
return true
|
||||
}
|
||||
return bestAllow
|
||||
}
|
||||
|
||||
// matchPattern matches a robots path pattern against path, honouring '*'
|
||||
// (any run) and a trailing '$' (end anchor).
|
||||
func matchPattern(pattern, path string) bool {
|
||||
anchored := strings.HasSuffix(pattern, "$")
|
||||
if anchored {
|
||||
pattern = pattern[:len(pattern)-1]
|
||||
}
|
||||
segs := strings.Split(pattern, "*")
|
||||
pos := 0
|
||||
for i, seg := range segs {
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
if i == 0 {
|
||||
if !strings.HasPrefix(path[pos:], seg) {
|
||||
return false
|
||||
}
|
||||
pos += len(seg)
|
||||
continue
|
||||
}
|
||||
idx := strings.Index(path[pos:], seg)
|
||||
if idx < 0 {
|
||||
return false
|
||||
}
|
||||
pos += idx + len(seg)
|
||||
}
|
||||
if anchored {
|
||||
// The last non-empty segment must land exactly at the end.
|
||||
last := ""
|
||||
for i := len(segs) - 1; i >= 0; i-- {
|
||||
if segs[i] != "" {
|
||||
last = segs[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if last != "" && !strings.HasSuffix(path, last) {
|
||||
return false
|
||||
}
|
||||
// A pattern with no trailing wildcard must consume the whole path.
|
||||
if !strings.HasSuffix(pattern, "*") && pos != len(path) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func stripComment(line string) string {
|
||||
if before, _, found := strings.Cut(line, "#"); found {
|
||||
return before
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func splitField(line string) (key, val string, ok bool) {
|
||||
k, v, found := strings.Cut(line, ":")
|
||||
if !found {
|
||||
return "", "", false
|
||||
}
|
||||
key = strings.TrimSpace(k)
|
||||
val = strings.TrimSpace(v)
|
||||
if key == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return key, val, true
|
||||
}
|
||||
|
||||
func parseDelay(val string) time.Duration {
|
||||
secs, err := strconv.ParseFloat(strings.TrimSpace(val), 64)
|
||||
if err != nil || secs <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(secs * float64(time.Second))
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package robots
|
||||
|
||||
import "testing"
|
||||
|
||||
const sample = `
|
||||
# example robots
|
||||
User-agent: *
|
||||
Disallow: /private/
|
||||
Allow: /private/public
|
||||
Disallow: /*.json$
|
||||
Crawl-delay: 2
|
||||
|
||||
User-agent: kage
|
||||
Disallow: /no-kage/
|
||||
|
||||
Sitemap: https://ex.com/sitemap.xml
|
||||
Sitemap: https://ex.com/news.xml
|
||||
`
|
||||
|
||||
func TestParseAndAllowedSpecificAgent(t *testing.T) {
|
||||
m := Parse(sample, "kage")
|
||||
// kage's own group only disallows /no-kage/.
|
||||
if m.Allowed("/private/secret") == false {
|
||||
t.Error("kage group should allow /private/secret (only /no-kage/ is blocked)")
|
||||
}
|
||||
if m.Allowed("/no-kage/x") {
|
||||
t.Error("/no-kage/x should be disallowed for kage")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowedWildcardGroup(t *testing.T) {
|
||||
m := Parse(sample, "somebot")
|
||||
cases := map[string]bool{
|
||||
"/": true,
|
||||
"/private/x": false, // Disallow /private/
|
||||
"/private/public": true, // longer Allow wins on tie/length
|
||||
"/data/file.json": false, // Disallow /*.json$
|
||||
"/data/file.jsonx": true, // $ anchors the extension
|
||||
"/ok": true,
|
||||
}
|
||||
for p, want := range cases {
|
||||
if got := m.Allowed(p); got != want {
|
||||
t.Errorf("Allowed(%q) = %v, want %v", p, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSitemapsAndDelay(t *testing.T) {
|
||||
m := Parse(sample, "somebot")
|
||||
if len(m.Sitemaps) != 2 {
|
||||
t.Fatalf("got %d sitemaps, want 2", len(m.Sitemaps))
|
||||
}
|
||||
if m.Sitemaps[0] != "https://ex.com/sitemap.xml" {
|
||||
t.Errorf("sitemap[0] = %q", m.Sitemaps[0])
|
||||
}
|
||||
if m.CrawlDelay.Seconds() != 2 {
|
||||
t.Errorf("crawl-delay = %v, want 2s", m.CrawlDelay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyDisallowAllowsAll(t *testing.T) {
|
||||
m := Parse("User-agent: *\nDisallow:\n", "x")
|
||||
if !m.Allowed("/anything") {
|
||||
t.Error("empty Disallow should allow everything")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoRobotsAllowsAll(t *testing.T) {
|
||||
m := AllowAll()
|
||||
if !m.Allowed("/whatever") {
|
||||
t.Error("AllowAll must allow everything")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user