From fafa3dfa516b4577caeff208494fe61b4ed7544b Mon Sep 17 00:00:00 2001 From: Duc-Tam Nguyen Date: Sun, 14 Jun 2026 20:17:18 +0700 Subject: [PATCH] Add the pack package: mirror to zim or self-contained binary BuildZIM walks a cloned host directory, turns every file into a content entry with a MIME inferred from its extension, picks a main page, and adds the standard metadata plus a mainPage redirect. state.json is skipped. BuildBinary appends the archive to a copy of kage behind a KAGEPCK1 trailer, and Embedded detects that trailer at startup so the binary serves itself. Handler maps / to the main page and /path to a content entry, the same handler the embedded viewer uses. --- pack/binary.go | 69 +++++++++++++ pack/embed.go | 50 +++++++++ pack/mime.go | 52 ++++++++++ pack/pack_test.go | 253 ++++++++++++++++++++++++++++++++++++++++++++++ pack/serve.go | 62 ++++++++++++ pack/zim.go | 244 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 730 insertions(+) create mode 100644 pack/binary.go create mode 100644 pack/embed.go create mode 100644 pack/mime.go create mode 100644 pack/pack_test.go create mode 100644 pack/serve.go create mode 100644 pack/zim.go diff --git a/pack/binary.go b/pack/binary.go new file mode 100644 index 0000000..1eb6da1 --- /dev/null +++ b/pack/binary.go @@ -0,0 +1,69 @@ +package pack + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" +) + +const ( + // trailerMagic brackets the appended-archive trailer at both ends, so a stray + // copy of it inside the base binary cannot be mistaken for a real trailer. + trailerMagic = "KAGEPCK1" + // trailerLen is magic + uint64 archive length + magic again. + trailerLen = len(trailerMagic) + 8 + len(trailerMagic) +) + +// BinaryOptions controls how a self-contained viewer is assembled. +type BinaryOptions struct { + Out string // output path + Base string // base kage binary; default os.Executable() +} + +// BuildBinary writes baseExe ++ zimBytes ++ trailer to opts.Out and marks it +// executable. The base must be a kage binary, since the viewer behaviour lives +// in kage's own startup hook (see Embedded); appending a ZIM to an arbitrary +// executable would only produce a broken file. It returns the output path and +// the total byte size. +func BuildBinary(zimBytes []byte, opts BinaryOptions) (string, int64, error) { + base := opts.Base + if base == "" { + exe, err := os.Executable() + if err != nil { + return "", 0, fmt.Errorf("pack: locate base binary: %w", err) + } + base = exe + } + if opts.Out == "" { + return "", 0, fmt.Errorf("pack: BuildBinary requires an output path") + } + + baseBytes, err := os.ReadFile(base) + if err != nil { + return "", 0, fmt.Errorf("pack: read base binary %q: %w", base, err) + } + + var tr bytes.Buffer + tr.WriteString(trailerMagic) + _ = binary.Write(&tr, binary.LittleEndian, uint64(len(zimBytes))) + tr.WriteString(trailerMagic) + + f, err := os.Create(opts.Out) + if err != nil { + return "", 0, err + } + for _, chunk := range [][]byte{baseBytes, zimBytes, tr.Bytes()} { + if _, err := f.Write(chunk); err != nil { + _ = f.Close() + return opts.Out, 0, err + } + } + if err := f.Close(); err != nil { + return opts.Out, 0, err + } + if err := os.Chmod(opts.Out, 0o755); err != nil { + return opts.Out, 0, err + } + return opts.Out, int64(len(baseBytes) + len(zimBytes) + trailerLen), nil +} diff --git a/pack/embed.go b/pack/embed.go new file mode 100644 index 0000000..a562bfa --- /dev/null +++ b/pack/embed.go @@ -0,0 +1,50 @@ +package pack + +import ( + "encoding/binary" + "io" + "os" +) + +// Embedded inspects the running executable for an appended ZIM archive. If the +// KAGEPCK1 trailer is present, it returns a ReaderAt bounded to the archive, its +// size, and ok=true; the file handle stays open for the life of the process so +// the viewer can serve from it. A normal kage build has no trailer, so the cost +// to every ordinary invocation is one Open plus a 24-byte ReadAt. +func Embedded() (ra io.ReaderAt, size int64, ok bool) { + exe, err := os.Executable() + if err != nil { + return nil, 0, false + } + f, err := os.Open(exe) + if err != nil { + return nil, 0, false + } + info, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, 0, false + } + total := info.Size() + if total < int64(trailerLen) { + _ = f.Close() + return nil, 0, false + } + + tr := make([]byte, trailerLen) + if _, err := f.ReadAt(tr, total-int64(trailerLen)); err != nil { + _ = f.Close() + return nil, 0, false + } + if string(tr[:8]) != trailerMagic || string(tr[trailerLen-8:]) != trailerMagic { + _ = f.Close() + return nil, 0, false + } + zlen := int64(binary.LittleEndian.Uint64(tr[8:16])) + start := total - int64(trailerLen) - zlen + if zlen <= 0 || start < 0 { + _ = f.Close() + return nil, 0, false + } + return io.NewSectionReader(f, start, zlen), zlen, true +} diff --git a/pack/mime.go b/pack/mime.go new file mode 100644 index 0000000..98ff872 --- /dev/null +++ b/pack/mime.go @@ -0,0 +1,52 @@ +package pack + +import ( + "path" + "strings" +) + +// mimeByExt maps a lower-case file extension (with the dot) to the MIME type +// kage records for it. Inference is by extension only, never by sniffing the +// bytes, so the same input always yields the same output. Anything not listed +// falls back to application/octet-stream and is stored uncompressed. +var mimeByExt = map[string]string{ + ".html": "text/html", + ".htm": "text/html", + ".css": "text/css", + ".js": "text/javascript", + ".mjs": "text/javascript", + ".json": "application/json", + ".xml": "application/xml", + ".svg": "image/svg+xml", + ".txt": "text/plain", + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".avif": "image/avif", + ".ico": "image/x-icon", + ".woff2": "font/woff2", + ".woff": "font/woff", + ".ttf": "font/ttf", + ".otf": "font/otf", + ".eot": "application/vnd.ms-fontobject", + ".mp4": "video/mp4", + ".m4v": "video/mp4", + ".webm": "video/webm", + ".mp3": "audio/mpeg", + ".ogg": "audio/ogg", + ".pdf": "application/pdf", + ".zip": "application/zip", + ".wasm": "application/wasm", +} + +// MimeForExt returns the MIME type for a path's extension, defaulting to +// application/octet-stream when the extension is unknown or absent. +func MimeForExt(p string) string { + ext := strings.ToLower(path.Ext(p)) + if m, ok := mimeByExt[ext]; ok { + return m + } + return "application/octet-stream" +} diff --git a/pack/pack_test.go b/pack/pack_test.go new file mode 100644 index 0000000..97a5ca5 --- /dev/null +++ b/pack/pack_test.go @@ -0,0 +1,253 @@ +package pack + +import ( + "bytes" + "encoding/binary" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/tamnd/kage/urlx" + "github.com/tamnd/kage/zim" +) + +// writeMirror lays down a small kage-style mirror under a temp dir and returns +// the host dir. +func writeMirror(t *testing.T) string { + t.Helper() + root := t.TempDir() + host := filepath.Join(root, "example.com") + files := map[string]string{ + "index.html": "Example Home

Hi

", + "about/index.html": "About

About

", + "_kage/example.com/x/logo.png": "\x89PNGfake", + "_kage/state.json": `{"visited":[]}`, // must be skipped + } + for rel, body := range files { + p := filepath.Join(host, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return host +} + +func TestMimeForExt(t *testing.T) { + cases := map[string]string{ + "a/b/index.html": "text/html", + "style.CSS": "text/css", + "data.json": "application/json", + "icon.svg": "image/svg+xml", + "logo.png": "image/png", + "photo.JPEG": "image/jpeg", + "font.woff2": "font/woff2", + "clip.mp4": "video/mp4", + "doc.pdf": "application/pdf", + "mystery": "application/octet-stream", + "archive.tar.zst": "application/octet-stream", + } + for in, want := range cases { + if got := MimeForExt(in); got != want { + t.Errorf("MimeForExt(%q) = %q, want %q", in, got, want) + } + } +} + +func TestBuildZIMRoundTrip(t *testing.T) { + host := writeMirror(t) + out := filepath.Join(t.TempDir(), "example.zim") + path, size, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", Version: "test"}) + if err != nil { + t.Fatalf("BuildZIM: %v", err) + } + if path != out { + t.Errorf("path = %q, want %q", path, out) + } + fi, err := os.Stat(out) + if err != nil { + t.Fatal(err) + } + if fi.Size() != size { + t.Errorf("reported size %d, file is %d", size, fi.Size()) + } + + r, err := zim.Open(out) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + + // Main page is the root index. + mp, err := r.MainPage() + if err != nil { + t.Fatalf("MainPage: %v", err) + } + if !bytes.Contains(mp.Data, []byte("Example Home")) { + t.Errorf("main page wrong: %.40q", mp.Data) + } + if mp.MimeType != "text/html" { + t.Errorf("main page mime = %q", mp.MimeType) + } + + // Binary asset round-trips byte-for-byte. + logo, err := r.Get(zim.NamespaceContent, "_kage/example.com/x/logo.png") + if err != nil { + t.Fatalf("Get logo: %v", err) + } + if string(logo.Data) != "\x89PNGfake" { + t.Errorf("logo bytes wrong: %q", logo.Data) + } + + // Title metadata comes from the main page's . + title, err := r.Get(zim.NamespaceMetadata, "Title") + if err != nil || string(title.Data) != "Example Home" { + t.Errorf("M/Title = %q, %v", title.Data, err) + } + + // state.json was skipped. + if _, err := r.Get(zim.NamespaceContent, urlx.DefaultReserved+"/state.json"); err == nil { + t.Error("state.json should not be packed") + } +} + +func TestBuildZIMDeterministic(t *testing.T) { + host := writeMirror(t) + dir := t.TempDir() + a, _, err := BuildZIM(host, ZIMOptions{Out: filepath.Join(dir, "a.zim"), Date: "2026-06-14"}) + if err != nil { + t.Fatal(err) + } + b, _, err := BuildZIM(host, ZIMOptions{Out: filepath.Join(dir, "b.zim"), Date: "2026-06-14"}) + if err != nil { + t.Fatal(err) + } + ba, _ := os.ReadFile(a) + bb, _ := os.ReadFile(b) + if !bytes.Equal(ba, bb) { + t.Error("same mirror produced different archives") + } +} + +func TestPickMainPage(t *testing.T) { + cases := []struct { + in []string + want string + }{ + {[]string{"a/index.html", "index.html", "b.html"}, "index.html"}, + {[]string{"z/deep/p.html", "top.html", "a/p.html"}, "top.html"}, + {[]string{"b/x.html", "a/x.html"}, "a/x.html"}, // same depth, lexical + {nil, ""}, + } + for _, c := range cases { + if got := pickMainPage(c.in); got != c.want { + t.Errorf("pickMainPage(%v) = %q, want %q", c.in, got, c.want) + } + } +} + +// TestBinaryTrailerRoundTrip exercises the BuildBinary append contract and the +// trailer it leaves, without depending on os.Executable: it appends a ZIM to a +// fake base, reads the trailer back the way Embedded does, and serves the +// recovered archive. +func TestBinaryTrailerRoundTrip(t *testing.T) { + host := writeMirror(t) + zbytes, err := BuildZIMBytes(host, ZIMOptions{Date: "2026-06-14"}) + if err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + base := filepath.Join(dir, "fakekage") + baseBytes := bytes.Repeat([]byte("BASE"), 64) // stand-in for a kage binary + if err := os.WriteFile(base, baseBytes, 0o755); err != nil { + t.Fatal(err) + } + + out := filepath.Join(dir, "viewer") + _, total, err := BuildBinary(zbytes, BinaryOptions{Out: out, Base: base}) + if err != nil { + t.Fatalf("BuildBinary: %v", err) + } + if total != int64(len(baseBytes)+len(zbytes)+trailerLen) { + t.Errorf("total %d, want %d", total, len(baseBytes)+len(zbytes)+trailerLen) + } + + f, err := os.Open(out) + if err != nil { + t.Fatal(err) + } + defer f.Close() + fi, _ := f.Stat() + end := fi.Size() + + tr := make([]byte, trailerLen) + if _, err := f.ReadAt(tr, end-int64(trailerLen)); err != nil { + t.Fatal(err) + } + if string(tr[:8]) != trailerMagic || string(tr[trailerLen-8:]) != trailerMagic { + t.Fatal("trailer magic missing") + } + zlen := int64(binary.LittleEndian.Uint64(tr[8:16])) + if zlen != int64(len(zbytes)) { + t.Errorf("trailer length %d, want %d", zlen, len(zbytes)) + } + start := end - int64(trailerLen) - zlen + if start != int64(len(baseBytes)) { + t.Errorf("archive start %d, want %d", start, len(baseBytes)) + } + + sec := io.NewSectionReader(f, start, zlen) + r, err := zim.NewReader(sec, zlen) + if err != nil { + t.Fatalf("reopen appended zim: %v", err) + } + mp, err := r.MainPage() + if err != nil || !bytes.Contains(mp.Data, []byte("Example Home")) { + t.Errorf("recovered main page wrong: %.40q (%v)", mp.Data, err) + } +} + +func TestHandler(t *testing.T) { + host := writeMirror(t) + out := filepath.Join(t.TempDir(), "h.zim") + if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14"}); err != nil { + t.Fatal(err) + } + r, err := zim.Open(out) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + srv := httptest.NewServer(Handler(r)) + defer srv.Close() + + get := func(p string) (int, string) { + resp, err := http.Get(srv.URL + p) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(b) + } + + if code, body := get("/"); code != 200 || !bytes.Contains([]byte(body), []byte("Example Home")) { + t.Errorf("GET / = %d %.30q", code, body) + } + if code, _ := get("/about/index.html"); code != 200 { + t.Errorf("GET /about/index.html = %d", code) + } + if code, _ := get("/" + urlx.DefaultReserved + "/state.json"); code != 404 { + t.Errorf("GET state.json = %d, want 404", code) + } + if code, _ := get("/missing.html"); code != 404 { + t.Errorf("GET missing = %d, want 404", code) + } +} diff --git a/pack/serve.go b/pack/serve.go new file mode 100644 index 0000000..6b5bfae --- /dev/null +++ b/pack/serve.go @@ -0,0 +1,62 @@ +package pack + +import ( + "errors" + "net/http" + "os/exec" + "runtime" + "strings" + + "github.com/tamnd/kage/zim" +) + +// Handler serves a ZIM archive over HTTP. "/" maps to the archive's main page; +// "/a/b.png" maps to the C/a/b.png content entry. Because the saved HTML's links +// are mirror-relative paths, and those are exactly the C urls, a click in a +// served page hits the right entry with no rewriting. A miss is a plain 404. +func Handler(r *zim.Reader) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + p := strings.TrimPrefix(req.URL.Path, "/") + if p == "" { + blob, err := r.MainPage() + if err != nil { + http.NotFound(w, req) + return + } + serveBlob(w, blob) + return + } + blob, err := r.Get(zim.NamespaceContent, p) + if errors.Is(err, zim.ErrNotFound) { + http.NotFound(w, req) + return + } + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + serveBlob(w, blob) + }) +} + +func serveBlob(w http.ResponseWriter, b zim.Blob) { + if b.MimeType != "" { + w.Header().Set("Content-Type", b.MimeType) + } + _, _ = w.Write(b.Data) +} + +// OpenInBrowser launches the platform default browser at url. It is best-effort: +// callers print the url regardless and never treat a launch failure as fatal. +func OpenInBrowser(url string) error { + var cmd *exec.Cmd + switch runtime.GOOS { + case "darwin": + cmd = exec.Command("open", url) + case "windows": + cmd = exec.Command("rundll32", "url.dll,FileProtocolHandler", url) + default: + cmd = exec.Command("xdg-open", url) + } + return cmd.Start() +} diff --git a/pack/zim.go b/pack/zim.go new file mode 100644 index 0000000..727204d --- /dev/null +++ b/pack/zim.go @@ -0,0 +1,244 @@ +// Package pack turns a kage mirror on disk into a distributable artifact: a ZIM +// archive, or a self-contained executable that serves the mirror offline. It is +// the only pack-side package that touches the filesystem and the running +// executable; the byte-level format work lives in the zim package. +package pack + +import ( + "bufio" + "bytes" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "golang.org/x/net/html" + + "github.com/tamnd/kage/urlx" + "github.com/tamnd/kage/zim" +) + +// ZIMOptions controls how a mirror is packed into a ZIM archive. Date is passed +// in from the CLI boundary rather than read from the clock, so the zim and pack +// packages stay pure and packing the same mirror twice is byte-identical. +type ZIMOptions struct { + Out string // output path (default <mirror-base>.zim) + NoCompress bool // store every cluster raw (code 1) + Title string // overrides M/Title + Description string // M/Description + Language string // M/Language (default "eng") + Date string // M/Date, e.g. "2026-06-14" + Version string // kage version, recorded as M/Scraper +} + +// BuildZIM walks mirrorDir, turns every file into a C/ content entry, infers the +// MIME from the extension, picks a main page, adds M/ metadata and a W/mainPage +// redirect, and writes a .zim to opts.Out. It returns the output path and the +// number of bytes written. +func BuildZIM(mirrorDir string, opts ZIMOptions) (string, int64, error) { + w, err := buildWriter(mirrorDir, opts) + if err != nil { + return "", 0, err + } + out := opts.Out + if out == "" { + out = filepath.Base(mirrorDir) + ".zim" + } + f, err := os.Create(out) + if err != nil { + return "", 0, err + } + bw := bufio.NewWriter(f) + n, err := w.WriteTo(bw) + if err != nil { + _ = f.Close() + return out, n, err + } + if err := bw.Flush(); err != nil { + _ = f.Close() + return out, n, err + } + return out, n, f.Close() +} + +// BuildZIMBytes is the buffer-returning sibling of BuildZIM: it runs the same +// walk and returns the archive in memory, which the binary path appends to a +// base executable without writing the ZIM to disk first. +func BuildZIMBytes(mirrorDir string, opts ZIMOptions) ([]byte, error) { + w, err := buildWriter(mirrorDir, opts) + if err != nil { + return nil, err + } + var buf bytes.Buffer + if _, err := w.WriteTo(&buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// buildWriter does the shared work of both BuildZIM and BuildZIMBytes: it loads +// every file under mirrorDir into a zim.Writer with metadata and a main page. +func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) { + info, err := os.Stat(mirrorDir) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("pack: %q is not a directory", mirrorDir) + } + + w := zim.NewWriter() + if opts.NoCompress { + w.SetNoCompress(true) + } + + skip := urlx.DefaultReserved + "/state.json" + var htmlPages []string + counts := map[string]int{} + + walkErr := filepath.WalkDir(mirrorDir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel := slashRel(mirrorDir, p) + if rel == skip { + return nil + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + mime := MimeForExt(rel) + if mime == "text/html" { + htmlPages = append(htmlPages, rel) + } + counts[mime]++ + w.AddContent(zim.NamespaceContent, rel, "", mime, data) + return nil + }) + if walkErr != nil { + return nil, walkErr + } + + main := pickMainPage(htmlPages) + if main != "" { + w.SetMainPage(zim.NamespaceContent, main) + w.AddRedirect(zim.NamespaceWellKnown, "mainPage", "", zim.NamespaceContent, main) + } + + host := filepath.Base(mirrorDir) + title := firstNonEmpty(opts.Title, htmlTitleOf(mirrorDir, main), host) + w.AddMetadata("Title", title) + w.AddMetadata("Language", firstNonEmpty(opts.Language, "eng")) + if opts.Description != "" { + w.AddMetadata("Description", opts.Description) + } + w.AddMetadata("Creator", "kage") + w.AddMetadata("Publisher", "kage") + if opts.Date != "" { + w.AddMetadata("Date", opts.Date) + } + w.AddMetadata("Scraper", strings.TrimSpace("kage "+opts.Version)) + w.AddMetadata("Source", host) + w.AddMetadata("Counter", counterString(counts)) + return w, nil +} + +// pickMainPage chooses the archive's entry point: the root index if present, +// else the shallowest HTML page, ties broken lexicographically for determinism. +// It returns "" when the mirror has no HTML at all. +func pickMainPage(htmlPages []string) string { + for _, p := range htmlPages { + if p == "index.html" { + return p + } + } + sorted := append([]string(nil), htmlPages...) + sort.Slice(sorted, func(i, j int) bool { + di, dj := strings.Count(sorted[i], "/"), strings.Count(sorted[j], "/") + if di != dj { + return di < dj + } + return sorted[i] < sorted[j] + }) + if len(sorted) > 0 { + return sorted[0] + } + return "" +} + +// htmlTitleOf reads the main page off disk and returns its <title>, or "" if +// there is no main page or no title. +func htmlTitleOf(mirrorDir, mainURL string) string { + if mainURL == "" { + return "" + } + f, err := os.Open(filepath.Join(mirrorDir, filepath.FromSlash(mainURL))) + if err != nil { + return "" + } + defer f.Close() + doc, err := html.Parse(f) + if err != nil { + return "" + } + return strings.TrimSpace(findTitle(doc)) +} + +// findTitle returns the text of the first <title> element in depth-first order. +func findTitle(n *html.Node) string { + if n.Type == html.ElementNode && n.Data == "title" { + var b strings.Builder + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.TextNode { + b.WriteString(c.Data) + } + } + return b.String() + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if t := findTitle(c); t != "" { + return t + } + } + return "" +} + +// counterString renders the M/Counter value Kiwix uses for stats: a +// semicolon-separated list of mime=count pairs, sorted for determinism. +func counterString(counts map[string]int) string { + mimes := make([]string, 0, len(counts)) + for m := range counts { + mimes = append(mimes, m) + } + sort.Strings(mimes) + parts := make([]string, len(mimes)) + for i, m := range mimes { + parts[i] = fmt.Sprintf("%s=%d", m, counts[m]) + } + return strings.Join(parts, ";") +} + +// slashRel returns p relative to root using forward slashes, the form ZIM urls +// take regardless of the host filesystem separator. +func slashRel(root, p string) string { + rel, err := filepath.Rel(root, p) + if err != nil { + rel = p + } + return filepath.ToSlash(rel) +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +}