Add the pack package: mirror to zim or self-contained binary
BuildZIM walks a cloned host directory, turns every file into a content entry with a MIME inferred from its extension, picks a main page, and adds the standard metadata plus a mainPage redirect. state.json is skipped. BuildBinary appends the archive to a copy of kage behind a KAGEPCK1 trailer, and Embedded detects that trailer at startup so the binary serves itself. Handler maps / to the main page and /path to a content entry, the same handler the embedded viewer uses.
This commit is contained in:
Duc-Tam Nguyen
@@ -0,0 +1,69 @@
|
||||
package pack
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
const (
|
||||
// trailerMagic brackets the appended-archive trailer at both ends, so a stray
|
||||
// copy of it inside the base binary cannot be mistaken for a real trailer.
|
||||
trailerMagic = "KAGEPCK1"
|
||||
// trailerLen is magic + uint64 archive length + magic again.
|
||||
trailerLen = len(trailerMagic) + 8 + len(trailerMagic)
|
||||
)
|
||||
|
||||
// BinaryOptions controls how a self-contained viewer is assembled.
|
||||
type BinaryOptions struct {
|
||||
Out string // output path
|
||||
Base string // base kage binary; default os.Executable()
|
||||
}
|
||||
|
||||
// BuildBinary writes baseExe ++ zimBytes ++ trailer to opts.Out and marks it
|
||||
// executable. The base must be a kage binary, since the viewer behaviour lives
|
||||
// in kage's own startup hook (see Embedded); appending a ZIM to an arbitrary
|
||||
// executable would only produce a broken file. It returns the output path and
|
||||
// the total byte size.
|
||||
func BuildBinary(zimBytes []byte, opts BinaryOptions) (string, int64, error) {
|
||||
base := opts.Base
|
||||
if base == "" {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("pack: locate base binary: %w", err)
|
||||
}
|
||||
base = exe
|
||||
}
|
||||
if opts.Out == "" {
|
||||
return "", 0, fmt.Errorf("pack: BuildBinary requires an output path")
|
||||
}
|
||||
|
||||
baseBytes, err := os.ReadFile(base)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("pack: read base binary %q: %w", base, err)
|
||||
}
|
||||
|
||||
var tr bytes.Buffer
|
||||
tr.WriteString(trailerMagic)
|
||||
_ = binary.Write(&tr, binary.LittleEndian, uint64(len(zimBytes)))
|
||||
tr.WriteString(trailerMagic)
|
||||
|
||||
f, err := os.Create(opts.Out)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
for _, chunk := range [][]byte{baseBytes, zimBytes, tr.Bytes()} {
|
||||
if _, err := f.Write(chunk); err != nil {
|
||||
_ = f.Close()
|
||||
return opts.Out, 0, err
|
||||
}
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return opts.Out, 0, err
|
||||
}
|
||||
if err := os.Chmod(opts.Out, 0o755); err != nil {
|
||||
return opts.Out, 0, err
|
||||
}
|
||||
return opts.Out, int64(len(baseBytes) + len(zimBytes) + trailerLen), nil
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package pack
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Embedded inspects the running executable for an appended ZIM archive. If the
|
||||
// KAGEPCK1 trailer is present, it returns a ReaderAt bounded to the archive, its
|
||||
// size, and ok=true; the file handle stays open for the life of the process so
|
||||
// the viewer can serve from it. A normal kage build has no trailer, so the cost
|
||||
// to every ordinary invocation is one Open plus a 24-byte ReadAt.
|
||||
func Embedded() (ra io.ReaderAt, size int64, ok bool) {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return nil, 0, false
|
||||
}
|
||||
f, err := os.Open(exe)
|
||||
if err != nil {
|
||||
return nil, 0, false
|
||||
}
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return nil, 0, false
|
||||
}
|
||||
total := info.Size()
|
||||
if total < int64(trailerLen) {
|
||||
_ = f.Close()
|
||||
return nil, 0, false
|
||||
}
|
||||
|
||||
tr := make([]byte, trailerLen)
|
||||
if _, err := f.ReadAt(tr, total-int64(trailerLen)); err != nil {
|
||||
_ = f.Close()
|
||||
return nil, 0, false
|
||||
}
|
||||
if string(tr[:8]) != trailerMagic || string(tr[trailerLen-8:]) != trailerMagic {
|
||||
_ = f.Close()
|
||||
return nil, 0, false
|
||||
}
|
||||
zlen := int64(binary.LittleEndian.Uint64(tr[8:16]))
|
||||
start := total - int64(trailerLen) - zlen
|
||||
if zlen <= 0 || start < 0 {
|
||||
_ = f.Close()
|
||||
return nil, 0, false
|
||||
}
|
||||
return io.NewSectionReader(f, start, zlen), zlen, true
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package pack
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// mimeByExt maps a lower-case file extension (with the dot) to the MIME type
|
||||
// kage records for it. Inference is by extension only, never by sniffing the
|
||||
// bytes, so the same input always yields the same output. Anything not listed
|
||||
// falls back to application/octet-stream and is stored uncompressed.
|
||||
var mimeByExt = map[string]string{
|
||||
".html": "text/html",
|
||||
".htm": "text/html",
|
||||
".css": "text/css",
|
||||
".js": "text/javascript",
|
||||
".mjs": "text/javascript",
|
||||
".json": "application/json",
|
||||
".xml": "application/xml",
|
||||
".svg": "image/svg+xml",
|
||||
".txt": "text/plain",
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
".avif": "image/avif",
|
||||
".ico": "image/x-icon",
|
||||
".woff2": "font/woff2",
|
||||
".woff": "font/woff",
|
||||
".ttf": "font/ttf",
|
||||
".otf": "font/otf",
|
||||
".eot": "application/vnd.ms-fontobject",
|
||||
".mp4": "video/mp4",
|
||||
".m4v": "video/mp4",
|
||||
".webm": "video/webm",
|
||||
".mp3": "audio/mpeg",
|
||||
".ogg": "audio/ogg",
|
||||
".pdf": "application/pdf",
|
||||
".zip": "application/zip",
|
||||
".wasm": "application/wasm",
|
||||
}
|
||||
|
||||
// MimeForExt returns the MIME type for a path's extension, defaulting to
|
||||
// application/octet-stream when the extension is unknown or absent.
|
||||
func MimeForExt(p string) string {
|
||||
ext := strings.ToLower(path.Ext(p))
|
||||
if m, ok := mimeByExt[ext]; ok {
|
||||
return m
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
package pack
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/tamnd/kage/urlx"
|
||||
"github.com/tamnd/kage/zim"
|
||||
)
|
||||
|
||||
// writeMirror lays down a small kage-style mirror under a temp dir and returns
|
||||
// the host dir.
|
||||
func writeMirror(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
host := filepath.Join(root, "example.com")
|
||||
files := map[string]string{
|
||||
"index.html": "<!doctype html><title>Example Home</title><h1>Hi</h1>",
|
||||
"about/index.html": "<!doctype html><title>About</title><h1>About</h1>",
|
||||
"_kage/example.com/x/logo.png": "\x89PNGfake",
|
||||
"_kage/state.json": `{"visited":[]}`, // must be skipped
|
||||
}
|
||||
for rel, body := range files {
|
||||
p := filepath.Join(host, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
func TestMimeForExt(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"a/b/index.html": "text/html",
|
||||
"style.CSS": "text/css",
|
||||
"data.json": "application/json",
|
||||
"icon.svg": "image/svg+xml",
|
||||
"logo.png": "image/png",
|
||||
"photo.JPEG": "image/jpeg",
|
||||
"font.woff2": "font/woff2",
|
||||
"clip.mp4": "video/mp4",
|
||||
"doc.pdf": "application/pdf",
|
||||
"mystery": "application/octet-stream",
|
||||
"archive.tar.zst": "application/octet-stream",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := MimeForExt(in); got != want {
|
||||
t.Errorf("MimeForExt(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildZIMRoundTrip(t *testing.T) {
|
||||
host := writeMirror(t)
|
||||
out := filepath.Join(t.TempDir(), "example.zim")
|
||||
path, size, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", Version: "test"})
|
||||
if err != nil {
|
||||
t.Fatalf("BuildZIM: %v", err)
|
||||
}
|
||||
if path != out {
|
||||
t.Errorf("path = %q, want %q", path, out)
|
||||
}
|
||||
fi, err := os.Stat(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fi.Size() != size {
|
||||
t.Errorf("reported size %d, file is %d", size, fi.Size())
|
||||
}
|
||||
|
||||
r, err := zim.Open(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
// Main page is the root index.
|
||||
mp, err := r.MainPage()
|
||||
if err != nil {
|
||||
t.Fatalf("MainPage: %v", err)
|
||||
}
|
||||
if !bytes.Contains(mp.Data, []byte("Example Home")) {
|
||||
t.Errorf("main page wrong: %.40q", mp.Data)
|
||||
}
|
||||
if mp.MimeType != "text/html" {
|
||||
t.Errorf("main page mime = %q", mp.MimeType)
|
||||
}
|
||||
|
||||
// Binary asset round-trips byte-for-byte.
|
||||
logo, err := r.Get(zim.NamespaceContent, "_kage/example.com/x/logo.png")
|
||||
if err != nil {
|
||||
t.Fatalf("Get logo: %v", err)
|
||||
}
|
||||
if string(logo.Data) != "\x89PNGfake" {
|
||||
t.Errorf("logo bytes wrong: %q", logo.Data)
|
||||
}
|
||||
|
||||
// Title metadata comes from the main page's <title>.
|
||||
title, err := r.Get(zim.NamespaceMetadata, "Title")
|
||||
if err != nil || string(title.Data) != "Example Home" {
|
||||
t.Errorf("M/Title = %q, %v", title.Data, err)
|
||||
}
|
||||
|
||||
// state.json was skipped.
|
||||
if _, err := r.Get(zim.NamespaceContent, urlx.DefaultReserved+"/state.json"); err == nil {
|
||||
t.Error("state.json should not be packed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildZIMDeterministic(t *testing.T) {
|
||||
host := writeMirror(t)
|
||||
dir := t.TempDir()
|
||||
a, _, err := BuildZIM(host, ZIMOptions{Out: filepath.Join(dir, "a.zim"), Date: "2026-06-14"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, _, err := BuildZIM(host, ZIMOptions{Out: filepath.Join(dir, "b.zim"), Date: "2026-06-14"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ba, _ := os.ReadFile(a)
|
||||
bb, _ := os.ReadFile(b)
|
||||
if !bytes.Equal(ba, bb) {
|
||||
t.Error("same mirror produced different archives")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickMainPage(t *testing.T) {
|
||||
cases := []struct {
|
||||
in []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"a/index.html", "index.html", "b.html"}, "index.html"},
|
||||
{[]string{"z/deep/p.html", "top.html", "a/p.html"}, "top.html"},
|
||||
{[]string{"b/x.html", "a/x.html"}, "a/x.html"}, // same depth, lexical
|
||||
{nil, ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := pickMainPage(c.in); got != c.want {
|
||||
t.Errorf("pickMainPage(%v) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBinaryTrailerRoundTrip exercises the BuildBinary append contract and the
|
||||
// trailer it leaves, without depending on os.Executable: it appends a ZIM to a
|
||||
// fake base, reads the trailer back the way Embedded does, and serves the
|
||||
// recovered archive.
|
||||
func TestBinaryTrailerRoundTrip(t *testing.T) {
|
||||
host := writeMirror(t)
|
||||
zbytes, err := BuildZIMBytes(host, ZIMOptions{Date: "2026-06-14"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
base := filepath.Join(dir, "fakekage")
|
||||
baseBytes := bytes.Repeat([]byte("BASE"), 64) // stand-in for a kage binary
|
||||
if err := os.WriteFile(base, baseBytes, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
out := filepath.Join(dir, "viewer")
|
||||
_, total, err := BuildBinary(zbytes, BinaryOptions{Out: out, Base: base})
|
||||
if err != nil {
|
||||
t.Fatalf("BuildBinary: %v", err)
|
||||
}
|
||||
if total != int64(len(baseBytes)+len(zbytes)+trailerLen) {
|
||||
t.Errorf("total %d, want %d", total, len(baseBytes)+len(zbytes)+trailerLen)
|
||||
}
|
||||
|
||||
f, err := os.Open(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
fi, _ := f.Stat()
|
||||
end := fi.Size()
|
||||
|
||||
tr := make([]byte, trailerLen)
|
||||
if _, err := f.ReadAt(tr, end-int64(trailerLen)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(tr[:8]) != trailerMagic || string(tr[trailerLen-8:]) != trailerMagic {
|
||||
t.Fatal("trailer magic missing")
|
||||
}
|
||||
zlen := int64(binary.LittleEndian.Uint64(tr[8:16]))
|
||||
if zlen != int64(len(zbytes)) {
|
||||
t.Errorf("trailer length %d, want %d", zlen, len(zbytes))
|
||||
}
|
||||
start := end - int64(trailerLen) - zlen
|
||||
if start != int64(len(baseBytes)) {
|
||||
t.Errorf("archive start %d, want %d", start, len(baseBytes))
|
||||
}
|
||||
|
||||
sec := io.NewSectionReader(f, start, zlen)
|
||||
r, err := zim.NewReader(sec, zlen)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen appended zim: %v", err)
|
||||
}
|
||||
mp, err := r.MainPage()
|
||||
if err != nil || !bytes.Contains(mp.Data, []byte("Example Home")) {
|
||||
t.Errorf("recovered main page wrong: %.40q (%v)", mp.Data, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler(t *testing.T) {
|
||||
host := writeMirror(t)
|
||||
out := filepath.Join(t.TempDir(), "h.zim")
|
||||
if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := zim.Open(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
srv := httptest.NewServer(Handler(r))
|
||||
defer srv.Close()
|
||||
|
||||
get := func(p string) (int, string) {
|
||||
resp, err := http.Get(srv.URL + p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return resp.StatusCode, string(b)
|
||||
}
|
||||
|
||||
if code, body := get("/"); code != 200 || !bytes.Contains([]byte(body), []byte("Example Home")) {
|
||||
t.Errorf("GET / = %d %.30q", code, body)
|
||||
}
|
||||
if code, _ := get("/about/index.html"); code != 200 {
|
||||
t.Errorf("GET /about/index.html = %d", code)
|
||||
}
|
||||
if code, _ := get("/" + urlx.DefaultReserved + "/state.json"); code != 404 {
|
||||
t.Errorf("GET state.json = %d, want 404", code)
|
||||
}
|
||||
if code, _ := get("/missing.html"); code != 404 {
|
||||
t.Errorf("GET missing = %d, want 404", code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package pack
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/tamnd/kage/zim"
|
||||
)
|
||||
|
||||
// Handler serves a ZIM archive over HTTP. "/" maps to the archive's main page;
|
||||
// "/a/b.png" maps to the C/a/b.png content entry. Because the saved HTML's links
|
||||
// are mirror-relative paths, and those are exactly the C urls, a click in a
|
||||
// served page hits the right entry with no rewriting. A miss is a plain 404.
|
||||
func Handler(r *zim.Reader) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
p := strings.TrimPrefix(req.URL.Path, "/")
|
||||
if p == "" {
|
||||
blob, err := r.MainPage()
|
||||
if err != nil {
|
||||
http.NotFound(w, req)
|
||||
return
|
||||
}
|
||||
serveBlob(w, blob)
|
||||
return
|
||||
}
|
||||
blob, err := r.Get(zim.NamespaceContent, p)
|
||||
if errors.Is(err, zim.ErrNotFound) {
|
||||
http.NotFound(w, req)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
serveBlob(w, blob)
|
||||
})
|
||||
}
|
||||
|
||||
func serveBlob(w http.ResponseWriter, b zim.Blob) {
|
||||
if b.MimeType != "" {
|
||||
w.Header().Set("Content-Type", b.MimeType)
|
||||
}
|
||||
_, _ = w.Write(b.Data)
|
||||
}
|
||||
|
||||
// OpenInBrowser launches the platform default browser at url. It is best-effort:
|
||||
// callers print the url regardless and never treat a launch failure as fatal.
|
||||
func OpenInBrowser(url string) error {
|
||||
var cmd *exec.Cmd
|
||||
switch runtime.GOOS {
|
||||
case "darwin":
|
||||
cmd = exec.Command("open", url)
|
||||
case "windows":
|
||||
cmd = exec.Command("rundll32", "url.dll,FileProtocolHandler", url)
|
||||
default:
|
||||
cmd = exec.Command("xdg-open", url)
|
||||
}
|
||||
return cmd.Start()
|
||||
}
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
// Package pack turns a kage mirror on disk into a distributable artifact: a ZIM
|
||||
// archive, or a self-contained executable that serves the mirror offline. It is
|
||||
// the only pack-side package that touches the filesystem and the running
|
||||
// executable; the byte-level format work lives in the zim package.
|
||||
package pack
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
|
||||
"github.com/tamnd/kage/urlx"
|
||||
"github.com/tamnd/kage/zim"
|
||||
)
|
||||
|
||||
// ZIMOptions controls how a mirror is packed into a ZIM archive. Date is passed
|
||||
// in from the CLI boundary rather than read from the clock, so the zim and pack
|
||||
// packages stay pure and packing the same mirror twice is byte-identical.
|
||||
type ZIMOptions struct {
|
||||
Out string // output path (default <mirror-base>.zim)
|
||||
NoCompress bool // store every cluster raw (code 1)
|
||||
Title string // overrides M/Title
|
||||
Description string // M/Description
|
||||
Language string // M/Language (default "eng")
|
||||
Date string // M/Date, e.g. "2026-06-14"
|
||||
Version string // kage version, recorded as M/Scraper
|
||||
}
|
||||
|
||||
// BuildZIM walks mirrorDir, turns every file into a C/ content entry, infers the
|
||||
// MIME from the extension, picks a main page, adds M/ metadata and a W/mainPage
|
||||
// redirect, and writes a .zim to opts.Out. It returns the output path and the
|
||||
// number of bytes written.
|
||||
func BuildZIM(mirrorDir string, opts ZIMOptions) (string, int64, error) {
|
||||
w, err := buildWriter(mirrorDir, opts)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
out := opts.Out
|
||||
if out == "" {
|
||||
out = filepath.Base(mirrorDir) + ".zim"
|
||||
}
|
||||
f, err := os.Create(out)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
bw := bufio.NewWriter(f)
|
||||
n, err := w.WriteTo(bw)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return out, n, err
|
||||
}
|
||||
if err := bw.Flush(); err != nil {
|
||||
_ = f.Close()
|
||||
return out, n, err
|
||||
}
|
||||
return out, n, f.Close()
|
||||
}
|
||||
|
||||
// BuildZIMBytes is the buffer-returning sibling of BuildZIM: it runs the same
|
||||
// walk and returns the archive in memory, which the binary path appends to a
|
||||
// base executable without writing the ZIM to disk first.
|
||||
func BuildZIMBytes(mirrorDir string, opts ZIMOptions) ([]byte, error) {
|
||||
w, err := buildWriter(mirrorDir, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if _, err := w.WriteTo(&buf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// buildWriter does the shared work of both BuildZIM and BuildZIMBytes: it loads
|
||||
// every file under mirrorDir into a zim.Writer with metadata and a main page.
|
||||
func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) {
|
||||
info, err := os.Stat(mirrorDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("pack: %q is not a directory", mirrorDir)
|
||||
}
|
||||
|
||||
w := zim.NewWriter()
|
||||
if opts.NoCompress {
|
||||
w.SetNoCompress(true)
|
||||
}
|
||||
|
||||
skip := urlx.DefaultReserved + "/state.json"
|
||||
var htmlPages []string
|
||||
counts := map[string]int{}
|
||||
|
||||
walkErr := filepath.WalkDir(mirrorDir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel := slashRel(mirrorDir, p)
|
||||
if rel == skip {
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mime := MimeForExt(rel)
|
||||
if mime == "text/html" {
|
||||
htmlPages = append(htmlPages, rel)
|
||||
}
|
||||
counts[mime]++
|
||||
w.AddContent(zim.NamespaceContent, rel, "", mime, data)
|
||||
return nil
|
||||
})
|
||||
if walkErr != nil {
|
||||
return nil, walkErr
|
||||
}
|
||||
|
||||
main := pickMainPage(htmlPages)
|
||||
if main != "" {
|
||||
w.SetMainPage(zim.NamespaceContent, main)
|
||||
w.AddRedirect(zim.NamespaceWellKnown, "mainPage", "", zim.NamespaceContent, main)
|
||||
}
|
||||
|
||||
host := filepath.Base(mirrorDir)
|
||||
title := firstNonEmpty(opts.Title, htmlTitleOf(mirrorDir, main), host)
|
||||
w.AddMetadata("Title", title)
|
||||
w.AddMetadata("Language", firstNonEmpty(opts.Language, "eng"))
|
||||
if opts.Description != "" {
|
||||
w.AddMetadata("Description", opts.Description)
|
||||
}
|
||||
w.AddMetadata("Creator", "kage")
|
||||
w.AddMetadata("Publisher", "kage")
|
||||
if opts.Date != "" {
|
||||
w.AddMetadata("Date", opts.Date)
|
||||
}
|
||||
w.AddMetadata("Scraper", strings.TrimSpace("kage "+opts.Version))
|
||||
w.AddMetadata("Source", host)
|
||||
w.AddMetadata("Counter", counterString(counts))
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// pickMainPage chooses the archive's entry point: the root index if present,
|
||||
// else the shallowest HTML page, ties broken lexicographically for determinism.
|
||||
// It returns "" when the mirror has no HTML at all.
|
||||
func pickMainPage(htmlPages []string) string {
|
||||
for _, p := range htmlPages {
|
||||
if p == "index.html" {
|
||||
return p
|
||||
}
|
||||
}
|
||||
sorted := append([]string(nil), htmlPages...)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
di, dj := strings.Count(sorted[i], "/"), strings.Count(sorted[j], "/")
|
||||
if di != dj {
|
||||
return di < dj
|
||||
}
|
||||
return sorted[i] < sorted[j]
|
||||
})
|
||||
if len(sorted) > 0 {
|
||||
return sorted[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// htmlTitleOf reads the main page off disk and returns its <title>, or "" if
|
||||
// there is no main page or no title.
|
||||
func htmlTitleOf(mirrorDir, mainURL string) string {
|
||||
if mainURL == "" {
|
||||
return ""
|
||||
}
|
||||
f, err := os.Open(filepath.Join(mirrorDir, filepath.FromSlash(mainURL)))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer f.Close()
|
||||
doc, err := html.Parse(f)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(findTitle(doc))
|
||||
}
|
||||
|
||||
// findTitle returns the text of the first <title> element in depth-first order.
|
||||
func findTitle(n *html.Node) string {
|
||||
if n.Type == html.ElementNode && n.Data == "title" {
|
||||
var b strings.Builder
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type == html.TextNode {
|
||||
b.WriteString(c.Data)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if t := findTitle(c); t != "" {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// counterString renders the M/Counter value Kiwix uses for stats: a
|
||||
// semicolon-separated list of mime=count pairs, sorted for determinism.
|
||||
func counterString(counts map[string]int) string {
|
||||
mimes := make([]string, 0, len(counts))
|
||||
for m := range counts {
|
||||
mimes = append(mimes, m)
|
||||
}
|
||||
sort.Strings(mimes)
|
||||
parts := make([]string, len(mimes))
|
||||
for i, m := range mimes {
|
||||
parts[i] = fmt.Sprintf("%s=%d", m, counts[m])
|
||||
}
|
||||
return strings.Join(parts, ";")
|
||||
}
|
||||
|
||||
// slashRel returns p relative to root using forward slashes, the form ZIM urls
|
||||
// take regardless of the host filesystem separator.
|
||||
func slashRel(root, p string) string {
|
||||
rel, err := filepath.Rel(root, p)
|
||||
if err != nil {
|
||||
rel = p
|
||||
}
|
||||
return filepath.ToSlash(rel)
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user