Add a pure-Go zim reader and writer
ZIM is the open single-file archive format Kiwix uses for offline content: a fixed header, a MIME list, URL/title/cluster pointer lists, directory entries, zstd-compressed or stored clusters, and a trailing MD5. The writer lays out a mirror in two passes (assign positions, then emit bytes) and derives the UUID from the content so packing is deterministic. The reader random-accesses entries by namespace and url, follows redirects, and reads xz clusters too so archives from other tooling open. Cross-checked against an independent reader (gozim): header, MIME list, namespaces, urls, dirents, and a non-last cluster's blob all read back byte-for-byte.
This commit is contained in:
Duc-Tam Nguyen
@@ -7,6 +7,7 @@ require (
|
||||
github.com/charmbracelet/fang v1.0.0
|
||||
github.com/go-rod/rod v0.116.2
|
||||
github.com/go-rod/stealth v0.4.9
|
||||
github.com/klauspost/compress v1.18.6
|
||||
github.com/spf13/cobra v1.10.2
|
||||
golang.org/x/net v0.56.0
|
||||
)
|
||||
|
||||
@@ -36,6 +36,8 @@ github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4=
|
||||
github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// A single shared zstd codec. Both EncodeAll and DecodeAll are safe for
|
||||
// concurrent use, so one encoder and one decoder serve the whole process.
|
||||
var (
|
||||
zstdOnce sync.Once
|
||||
zstdEnc *zstd.Encoder
|
||||
zstdDec *zstd.Decoder
|
||||
)
|
||||
|
||||
func initZstd() {
|
||||
zstdOnce.Do(func() {
|
||||
zstdEnc, _ = zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
|
||||
zstdDec, _ = zstd.NewReader(nil)
|
||||
})
|
||||
}
|
||||
|
||||
func zstdEncode(p []byte) []byte {
|
||||
initZstd()
|
||||
return zstdEnc.EncodeAll(p, nil)
|
||||
}
|
||||
|
||||
func zstdDecode(p []byte) ([]byte, error) {
|
||||
initZstd()
|
||||
return zstdDec.DecodeAll(p, nil)
|
||||
}
|
||||
|
||||
// isTextMime reports whether content of this MIME type is worth compressing.
|
||||
// Already-compressed media (images, fonts, audio, video, archives) is stored
|
||||
// uncompressed so we do not burn CPU inflating it by a few bytes.
|
||||
func isTextMime(mime string) bool {
|
||||
switch mime {
|
||||
case "application/json", "application/xml", "application/javascript",
|
||||
"application/x-javascript":
|
||||
return true
|
||||
}
|
||||
if len(mime) >= 5 && mime[:5] == "text/" {
|
||||
return true
|
||||
}
|
||||
// Any structured-XML type: application/rss+xml, image/svg+xml, ...
|
||||
return len(mime) >= 4 && mime[len(mime)-4:] == "+xml"
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
// Package zim reads and writes the ZIM offline-archive format, the open
|
||||
// single-file container that Kiwix uses to ship offline content. kage uses it
|
||||
// to pack a cloned mirror into one indexable, compressed file that a reader can
|
||||
// random-access without unpacking.
|
||||
//
|
||||
// The package is pure: no network, no clock, no global state beyond a lazily
|
||||
// built zstd codec. A ZIM file is laid out as a fixed header, a MIME-type list,
|
||||
// three pointer lists (URL, title, cluster), a run of directory entries, a run
|
||||
// of clusters that hold the content, and a trailing MD5. Every cross-reference
|
||||
// is an absolute file position recorded in the header, so the writer assigns
|
||||
// positions in one pass and emits bytes in a second. All integers are
|
||||
// little-endian.
|
||||
//
|
||||
// We write the new namespace scheme (minor version 1): all content lives under
|
||||
// the single 'C' namespace, metadata under 'M', and a 'W/mainPage' redirect
|
||||
// points at the entry point. Reading handles redirects and both offset widths.
|
||||
package zim
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Magic is the ZIM header magic number, the first four bytes of every file.
|
||||
const Magic uint32 = 0x44D495A // 72173914
|
||||
|
||||
const (
|
||||
majorVersion uint16 = 5
|
||||
minorVersion uint16 = 1 // single 'C' content namespace
|
||||
headerLen = 80
|
||||
)
|
||||
|
||||
// Namespaces in the new (minor version 1) scheme.
|
||||
const (
|
||||
NamespaceContent byte = 'C' // pages and assets
|
||||
NamespaceMetadata byte = 'M' // M/Title, M/Date, ...
|
||||
NamespaceWellKnown byte = 'W' // W/mainPage redirect
|
||||
)
|
||||
|
||||
// Compression codes carried in the low nibble of a cluster's info byte.
|
||||
const (
|
||||
compNone uint8 = 1 // stored, no compression
|
||||
compXZ uint8 = 4 // xz / LZMA2 (read-only support)
|
||||
compZstd uint8 = 5 // zstd (what we write for text)
|
||||
|
||||
extendedFlag uint8 = 0x10 // bit 4: cluster offsets are uint64, not uint32
|
||||
)
|
||||
|
||||
// Sentinels stored in a directory entry's mimetype field to mark non-content
|
||||
// entries. A redirect reuses the cluster slot to hold its target's URL index.
|
||||
const (
|
||||
redirectEntry uint16 = 0xffff
|
||||
linkTargetEntry uint16 = 0xfffe
|
||||
deletedEntry uint16 = 0xfffd
|
||||
)
|
||||
|
||||
// noMainPage is the mainPage/layoutPage value meaning "none".
|
||||
const noMainPage uint32 = 0xffffffff
|
||||
|
||||
// header is the 80-byte ZIM header.
|
||||
type header struct {
|
||||
uuid [16]byte
|
||||
articleCount uint32
|
||||
clusterCount uint32
|
||||
urlPtrPos uint64
|
||||
titlePtrPos uint64
|
||||
clusterPtrPos uint64
|
||||
mimeListPos uint64
|
||||
mainPage uint32
|
||||
layoutPage uint32
|
||||
checksumPos uint64
|
||||
}
|
||||
|
||||
// marshal encodes the header to its 80 wire bytes.
|
||||
func (h header) marshal() []byte {
|
||||
b := make([]byte, headerLen)
|
||||
le := binary.LittleEndian
|
||||
le.PutUint32(b[0:], Magic)
|
||||
le.PutUint16(b[4:], majorVersion)
|
||||
le.PutUint16(b[6:], minorVersion)
|
||||
copy(b[8:24], h.uuid[:])
|
||||
le.PutUint32(b[24:], h.articleCount)
|
||||
le.PutUint32(b[28:], h.clusterCount)
|
||||
le.PutUint64(b[32:], h.urlPtrPos)
|
||||
le.PutUint64(b[40:], h.titlePtrPos)
|
||||
le.PutUint64(b[48:], h.clusterPtrPos)
|
||||
le.PutUint64(b[56:], h.mimeListPos)
|
||||
le.PutUint32(b[64:], h.mainPage)
|
||||
le.PutUint32(b[68:], h.layoutPage)
|
||||
le.PutUint64(b[72:], h.checksumPos)
|
||||
return b
|
||||
}
|
||||
|
||||
// parseHeader decodes and validates an 80-byte header.
|
||||
func parseHeader(b []byte) (header, error) {
|
||||
var h header
|
||||
if len(b) < headerLen {
|
||||
return h, fmt.Errorf("zim: short header: %d bytes", len(b))
|
||||
}
|
||||
le := binary.LittleEndian
|
||||
if le.Uint32(b[0:]) != Magic {
|
||||
return h, fmt.Errorf("zim: bad magic, not a ZIM file")
|
||||
}
|
||||
copy(h.uuid[:], b[8:24])
|
||||
h.articleCount = le.Uint32(b[24:])
|
||||
h.clusterCount = le.Uint32(b[28:])
|
||||
h.urlPtrPos = le.Uint64(b[32:])
|
||||
h.titlePtrPos = le.Uint64(b[40:])
|
||||
h.clusterPtrPos = le.Uint64(b[48:])
|
||||
h.mimeListPos = le.Uint64(b[56:])
|
||||
h.mainPage = le.Uint32(b[64:])
|
||||
h.layoutPage = le.Uint32(b[68:])
|
||||
h.checksumPos = le.Uint64(b[72:])
|
||||
return h, nil
|
||||
}
|
||||
+339
@@ -0,0 +1,339 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// ErrNotFound is returned by Get when no entry matches the namespace and url.
|
||||
// Callers (such as the HTTP handler) test for it with errors.Is to map a miss
|
||||
// to a 404.
|
||||
var ErrNotFound = errors.New("zim: not found")
|
||||
|
||||
// Reader provides random access to a ZIM file's entries. Open one with Open or
|
||||
// NewReader, then look entries up by namespace and url, or fetch the main page.
|
||||
// Decompressed clusters are cached so repeated reads from one cluster are cheap.
|
||||
type Reader struct {
|
||||
ra io.ReaderAt
|
||||
closer io.Closer
|
||||
size int64
|
||||
hdr header
|
||||
mimes []string
|
||||
|
||||
mu sync.Mutex
|
||||
cache map[uint32][]byte // cluster index -> decompressed data section
|
||||
cacheExtended map[uint32]bool // cluster index -> uint64-offset cluster
|
||||
}
|
||||
|
||||
// Blob is the result of a lookup: the resolved entry's bytes and metadata.
|
||||
type Blob struct {
|
||||
Namespace byte
|
||||
URL string
|
||||
Title string
|
||||
MimeType string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// Open opens a ZIM file on disk. Close the returned reader when done.
|
||||
func Open(path string) (*Reader, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return nil, err
|
||||
}
|
||||
r, err := NewReader(f, fi.Size())
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return nil, err
|
||||
}
|
||||
r.closer = f
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// NewReader reads the header and MIME list from ra, which must hold size bytes.
|
||||
func NewReader(ra io.ReaderAt, size int64) (*Reader, error) {
|
||||
r := &Reader{ra: ra, size: size, cache: map[uint32][]byte{}}
|
||||
hb, err := r.at(0, headerLen)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("zim: read header: %w", err)
|
||||
}
|
||||
r.hdr, err = parseHeader(hb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.hdr.mimeListPos > r.hdr.urlPtrPos || r.hdr.urlPtrPos > uint64(size) {
|
||||
return nil, fmt.Errorf("zim: inconsistent header offsets")
|
||||
}
|
||||
mb, err := r.at(r.hdr.mimeListPos, int(r.hdr.urlPtrPos-r.hdr.mimeListPos))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("zim: read mime list: %w", err)
|
||||
}
|
||||
for _, part := range bytes.Split(mb, []byte{0}) {
|
||||
if len(part) == 0 {
|
||||
break
|
||||
}
|
||||
r.mimes = append(r.mimes, string(part))
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Close releases the underlying file, if Open created one.
|
||||
func (r *Reader) Close() error {
|
||||
if r.closer != nil {
|
||||
return r.closer.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Count returns the number of directory entries.
|
||||
func (r *Reader) Count() uint32 { return r.hdr.articleCount }
|
||||
|
||||
// MimeTypes returns the archive's MIME-type list.
|
||||
func (r *Reader) MimeTypes() []string { return r.mimes }
|
||||
|
||||
// MainPage returns the archive's entry point, or an error if none is set.
|
||||
func (r *Reader) MainPage() (Blob, error) {
|
||||
if r.hdr.mainPage == noMainPage {
|
||||
return Blob{}, fmt.Errorf("zim: no main page")
|
||||
}
|
||||
return r.blobAtIndex(r.hdr.mainPage, 0)
|
||||
}
|
||||
|
||||
// Get resolves the entry at (namespace, url), following one or more redirects.
|
||||
func (r *Reader) Get(namespace byte, url string) (Blob, error) {
|
||||
target := key(namespace, url)
|
||||
lo, hi := uint32(0), r.hdr.articleCount
|
||||
for lo < hi {
|
||||
mid := lo + (hi-lo)/2
|
||||
d, err := r.direntAtIndex(mid)
|
||||
if err != nil {
|
||||
return Blob{}, err
|
||||
}
|
||||
switch k := key(d.namespace, d.url); {
|
||||
case k < target:
|
||||
lo = mid + 1
|
||||
case k > target:
|
||||
hi = mid
|
||||
default:
|
||||
return r.blobAtIndex(mid, 0)
|
||||
}
|
||||
}
|
||||
return Blob{}, fmt.Errorf("%w: %c/%s", ErrNotFound, namespace, url)
|
||||
}
|
||||
|
||||
const maxRedirectHops = 16
|
||||
|
||||
func (r *Reader) blobAtIndex(idx uint32, hop int) (Blob, error) {
|
||||
if hop > maxRedirectHops {
|
||||
return Blob{}, fmt.Errorf("zim: redirect loop")
|
||||
}
|
||||
d, err := r.direntAtIndex(idx)
|
||||
if err != nil {
|
||||
return Blob{}, err
|
||||
}
|
||||
if d.redirect {
|
||||
return r.blobAtIndex(d.targetIndex, hop+1)
|
||||
}
|
||||
data, err := r.blobData(d.cluster, d.blob)
|
||||
if err != nil {
|
||||
return Blob{}, err
|
||||
}
|
||||
mime := ""
|
||||
if int(d.mimeIdx) < len(r.mimes) {
|
||||
mime = r.mimes[d.mimeIdx]
|
||||
}
|
||||
return Blob{Namespace: d.namespace, URL: d.url, Title: d.title, MimeType: mime, Data: data}, nil
|
||||
}
|
||||
|
||||
type dirent struct {
|
||||
mimeIdx uint16
|
||||
namespace byte
|
||||
url, title string
|
||||
cluster uint32
|
||||
blob uint32
|
||||
redirect bool
|
||||
targetIndex uint32
|
||||
}
|
||||
|
||||
func (r *Reader) direntAtIndex(idx uint32) (dirent, error) {
|
||||
pb, err := r.at(r.hdr.urlPtrPos+8*uint64(idx), 8)
|
||||
if err != nil {
|
||||
return dirent{}, err
|
||||
}
|
||||
return r.direntAt(binary.LittleEndian.Uint64(pb))
|
||||
}
|
||||
|
||||
func (r *Reader) direntAt(off uint64) (dirent, error) {
|
||||
// Read a window large enough for the fixed head plus url and title; grow if
|
||||
// either string is not terminated within it.
|
||||
window := 512
|
||||
for {
|
||||
b, err := r.at(off, window)
|
||||
if err != nil && len(b) == 0 {
|
||||
return dirent{}, err
|
||||
}
|
||||
var d dirent
|
||||
le := binary.LittleEndian
|
||||
d.mimeIdx = le.Uint16(b[0:])
|
||||
d.namespace = b[3]
|
||||
var p int
|
||||
if d.mimeIdx == redirectEntry {
|
||||
d.redirect = true
|
||||
d.targetIndex = le.Uint32(b[8:])
|
||||
p = 12
|
||||
} else {
|
||||
d.cluster = le.Uint32(b[8:])
|
||||
d.blob = le.Uint32(b[12:])
|
||||
p = 16
|
||||
}
|
||||
url, n1, ok := readCString(b, p)
|
||||
if !ok {
|
||||
if window >= 1<<20 || off+uint64(window) >= uint64(r.size) {
|
||||
return dirent{}, fmt.Errorf("zim: unterminated url at %d", off)
|
||||
}
|
||||
window *= 4
|
||||
continue
|
||||
}
|
||||
title, _, ok := readCString(b, n1)
|
||||
if !ok {
|
||||
if window >= 1<<20 || off+uint64(window) >= uint64(r.size) {
|
||||
return dirent{}, fmt.Errorf("zim: unterminated title at %d", off)
|
||||
}
|
||||
window *= 4
|
||||
continue
|
||||
}
|
||||
d.url, d.title = url, title
|
||||
return d, nil
|
||||
}
|
||||
}
|
||||
|
||||
// blobData returns one blob's bytes, decompressing and caching its cluster.
|
||||
func (r *Reader) blobData(cluster, blob uint32) ([]byte, error) {
|
||||
data, extended, err := r.clusterData(cluster)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w := uint32(4)
|
||||
if extended {
|
||||
w = 8
|
||||
}
|
||||
need := int((blob + 2) * w)
|
||||
if need > len(data) {
|
||||
return nil, fmt.Errorf("zim: blob %d out of range in cluster %d", blob, cluster)
|
||||
}
|
||||
o0 := readUint(data[blob*w:], w)
|
||||
o1 := readUint(data[(blob+1)*w:], w)
|
||||
if o0 > o1 || int(o1) > len(data) {
|
||||
return nil, fmt.Errorf("zim: bad blob offsets in cluster %d", cluster)
|
||||
}
|
||||
out := make([]byte, o1-o0)
|
||||
copy(out, data[o0:o1])
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *Reader) clusterData(cluster uint32) (data []byte, extended bool, err error) {
|
||||
r.mu.Lock()
|
||||
if c, ok := r.cache[cluster]; ok {
|
||||
r.mu.Unlock()
|
||||
// extended-ness is recoverable from the info byte, but the cache stores
|
||||
// already-decoded data whose offsets we re-read with the recorded width.
|
||||
return c, r.cacheExtended[cluster], nil
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
start, err := r.clusterOffset(cluster)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
end := r.hdr.checksumPos
|
||||
if cluster+1 < r.hdr.clusterCount {
|
||||
if end, err = r.clusterOffset(cluster + 1); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
if start >= end || end > uint64(r.size) {
|
||||
return nil, false, fmt.Errorf("zim: bad cluster bounds for %d", cluster)
|
||||
}
|
||||
raw, err := r.at(start, int(end-start))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
info := raw[0]
|
||||
comp := info & 0x0f
|
||||
extended = info&extendedFlag != 0
|
||||
body := raw[1:]
|
||||
switch comp {
|
||||
case compNone:
|
||||
data = body
|
||||
case compZstd:
|
||||
if data, err = zstdDecode(body); err != nil {
|
||||
return nil, false, fmt.Errorf("zim: zstd cluster %d: %w", cluster, err)
|
||||
}
|
||||
case compXZ:
|
||||
return nil, false, fmt.Errorf("zim: xz clusters are not supported for reading")
|
||||
default:
|
||||
return nil, false, fmt.Errorf("zim: unknown compression %d in cluster %d", comp, cluster)
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.cache[cluster] = data
|
||||
if r.cacheExtended == nil {
|
||||
r.cacheExtended = map[uint32]bool{}
|
||||
}
|
||||
r.cacheExtended[cluster] = extended
|
||||
r.mu.Unlock()
|
||||
return data, extended, nil
|
||||
}
|
||||
|
||||
func (r *Reader) clusterOffset(cluster uint32) (uint64, error) {
|
||||
b, err := r.at(r.hdr.clusterPtrPos+8*uint64(cluster), 8)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return binary.LittleEndian.Uint64(b), nil
|
||||
}
|
||||
|
||||
// at reads n bytes at off, clamped to the file size.
|
||||
func (r *Reader) at(off uint64, n int) ([]byte, error) {
|
||||
if n < 0 {
|
||||
return nil, fmt.Errorf("zim: negative read length")
|
||||
}
|
||||
if off > uint64(r.size) {
|
||||
return nil, io.EOF
|
||||
}
|
||||
if off+uint64(n) > uint64(r.size) {
|
||||
n = int(uint64(r.size) - off)
|
||||
}
|
||||
b := make([]byte, n)
|
||||
if n == 0 {
|
||||
return b, nil
|
||||
}
|
||||
_, err := r.ra.ReadAt(b, int64(off))
|
||||
return b, err
|
||||
}
|
||||
|
||||
func readCString(b []byte, start int) (string, int, bool) {
|
||||
if start > len(b) {
|
||||
return "", start, false
|
||||
}
|
||||
i := bytes.IndexByte(b[start:], 0)
|
||||
if i < 0 {
|
||||
return "", start, false
|
||||
}
|
||||
return string(b[start : start+i]), start + i + 1, true
|
||||
}
|
||||
|
||||
func readUint(b []byte, width uint32) uint32 {
|
||||
if width == 8 {
|
||||
return uint32(binary.LittleEndian.Uint64(b))
|
||||
}
|
||||
return binary.LittleEndian.Uint32(b)
|
||||
}
|
||||
+392
@@ -0,0 +1,392 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// maxClusterContent caps how much blob content accumulates in one cluster
|
||||
// before a new one is started, balancing compression ratio against the cost of
|
||||
// decompressing a whole cluster to read one small blob.
|
||||
const maxClusterContent = 2 << 20 // 2 MiB
|
||||
|
||||
// Writer accumulates entries and serialises them as a ZIM file. Build it with
|
||||
// NewWriter, add content/redirects/metadata, optionally set a main page, then
|
||||
// call WriteTo. The writer holds entries in memory; a kage mirror comfortably
|
||||
// fits, and packing is a one-shot batch job.
|
||||
type Writer struct {
|
||||
entries []*entry
|
||||
byKey map[string]*entry
|
||||
mainKey string
|
||||
noCompress bool
|
||||
}
|
||||
|
||||
type entry struct {
|
||||
namespace byte
|
||||
url string
|
||||
title string
|
||||
mime string
|
||||
data []byte
|
||||
|
||||
redirect bool
|
||||
targetKey string // "<ns><url>" of the redirect target
|
||||
|
||||
// assigned during planning
|
||||
mimeIdx uint16
|
||||
cluster uint32
|
||||
blob uint32
|
||||
targetIndex uint32
|
||||
urlIndex uint32
|
||||
position uint64
|
||||
}
|
||||
|
||||
func key(ns byte, url string) string { return string(ns) + url }
|
||||
|
||||
// NewWriter returns an empty Writer.
|
||||
func NewWriter() *Writer {
|
||||
return &Writer{byKey: map[string]*entry{}}
|
||||
}
|
||||
|
||||
// SetNoCompress stores every cluster uncompressed. Useful when the input is
|
||||
// already compressed or when a reader without zstd must open the file.
|
||||
func (w *Writer) SetNoCompress(v bool) { w.noCompress = v }
|
||||
|
||||
// AddContent adds a content entry. A later add with the same namespace and url
|
||||
// replaces the earlier one. An empty title defaults to the url.
|
||||
func (w *Writer) AddContent(namespace byte, url, title, mime string, data []byte) {
|
||||
if title == "" {
|
||||
title = url
|
||||
}
|
||||
if mime == "" {
|
||||
mime = "application/octet-stream"
|
||||
}
|
||||
w.put(&entry{namespace: namespace, url: url, title: title, mime: mime, data: data})
|
||||
}
|
||||
|
||||
// AddMetadata adds an 'M' namespace text entry, e.g. AddMetadata("Title", "...").
|
||||
func (w *Writer) AddMetadata(name, value string) {
|
||||
w.put(&entry{namespace: NamespaceMetadata, url: name, title: name, mime: "text/plain", data: []byte(value)})
|
||||
}
|
||||
|
||||
// AddRedirect adds a redirect from (namespace,url) to (targetNamespace,targetURL).
|
||||
func (w *Writer) AddRedirect(namespace byte, url, title string, targetNamespace byte, targetURL string) {
|
||||
if title == "" {
|
||||
title = url
|
||||
}
|
||||
w.put(&entry{namespace: namespace, url: url, title: title, redirect: true, targetKey: key(targetNamespace, targetURL)})
|
||||
}
|
||||
|
||||
// SetMainPage marks an entry as the archive's entry point.
|
||||
func (w *Writer) SetMainPage(namespace byte, url string) { w.mainKey = key(namespace, url) }
|
||||
|
||||
func (w *Writer) put(e *entry) {
|
||||
k := key(e.namespace, e.url)
|
||||
if old, ok := w.byKey[k]; ok {
|
||||
*old = *e // replace in place, keep slice order
|
||||
return
|
||||
}
|
||||
w.byKey[k] = e
|
||||
w.entries = append(w.entries, e)
|
||||
}
|
||||
|
||||
// plan holds the prebuilt sections of the file, ready to emit in order.
|
||||
type plan struct {
|
||||
hdr header
|
||||
mimeList []byte
|
||||
urlPtrs []byte
|
||||
titlePtrs []byte
|
||||
clusterPtrs []byte
|
||||
dirents [][]byte // URL order
|
||||
clusters [][]byte
|
||||
}
|
||||
|
||||
// WriteTo serialises the archive to out and returns the number of bytes written.
|
||||
func (w *Writer) WriteTo(out io.Writer) (int64, error) {
|
||||
p, err := w.buildPlan()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
sum := md5.New()
|
||||
mw := io.MultiWriter(out, sum)
|
||||
var n int64
|
||||
write := func(b []byte) error {
|
||||
m, err := mw.Write(b)
|
||||
n += int64(m)
|
||||
return err
|
||||
}
|
||||
for _, section := range append([][]byte{
|
||||
p.hdr.marshal(), p.mimeList, p.urlPtrs, p.titlePtrs, p.clusterPtrs,
|
||||
}, append(p.dirents, p.clusters...)...) {
|
||||
if err := write(section); err != nil {
|
||||
return n, err
|
||||
}
|
||||
}
|
||||
// The MD5 covers everything before it and is not itself hashed.
|
||||
m, err := out.Write(sum.Sum(nil))
|
||||
n += int64(m)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (w *Writer) buildPlan() (plan, error) {
|
||||
var p plan
|
||||
// 1. URL order: sort by <namespace><url>, assign indices.
|
||||
ents := make([]*entry, len(w.entries))
|
||||
copy(ents, w.entries)
|
||||
sort.Slice(ents, func(i, j int) bool {
|
||||
return key(ents[i].namespace, ents[i].url) < key(ents[j].namespace, ents[j].url)
|
||||
})
|
||||
index := make(map[string]uint32, len(ents))
|
||||
for i, e := range ents {
|
||||
e.urlIndex = uint32(i)
|
||||
index[key(e.namespace, e.url)] = uint32(i)
|
||||
}
|
||||
|
||||
// 2. Resolve redirect targets.
|
||||
for _, e := range ents {
|
||||
if !e.redirect {
|
||||
continue
|
||||
}
|
||||
ti, ok := index[e.targetKey]
|
||||
if !ok {
|
||||
return p, fmt.Errorf("zim: redirect %q points at missing target %q", key(e.namespace, e.url), e.targetKey)
|
||||
}
|
||||
e.targetIndex = ti
|
||||
}
|
||||
|
||||
// 3. MIME list (first-seen order over content entries).
|
||||
var mimes []string
|
||||
mimeIndex := map[string]uint16{}
|
||||
for _, e := range ents {
|
||||
if e.redirect {
|
||||
continue
|
||||
}
|
||||
if _, ok := mimeIndex[e.mime]; !ok {
|
||||
mimeIndex[e.mime] = uint16(len(mimes))
|
||||
mimes = append(mimes, e.mime)
|
||||
}
|
||||
e.mimeIdx = mimeIndex[e.mime]
|
||||
}
|
||||
p.mimeList = encodeMimeList(mimes)
|
||||
|
||||
// 4. Cluster packing: split text vs binary, cap each cluster, assign blobs.
|
||||
clusters := w.packClusters(ents)
|
||||
p.clusters = make([][]byte, len(clusters))
|
||||
for i, c := range clusters {
|
||||
p.clusters[i] = c.encode(w.noCompress)
|
||||
}
|
||||
|
||||
// 5. Directory entry bytes (URL order).
|
||||
p.dirents = make([][]byte, len(ents))
|
||||
for i, e := range ents {
|
||||
p.dirents[i] = e.encodeDirent()
|
||||
}
|
||||
|
||||
// 6. Layout: assign absolute positions.
|
||||
count := uint32(len(ents))
|
||||
pos := uint64(headerLen)
|
||||
mimeListPos := pos
|
||||
pos += uint64(len(p.mimeList))
|
||||
urlPtrPos := pos
|
||||
pos += 8 * uint64(count)
|
||||
titlePtrPos := pos
|
||||
pos += 4 * uint64(count)
|
||||
clusterPtrPos := pos
|
||||
pos += 8 * uint64(len(p.clusters))
|
||||
for i, e := range ents {
|
||||
e.position = pos
|
||||
pos += uint64(len(p.dirents[i]))
|
||||
}
|
||||
clusterPos := make([]uint64, len(p.clusters))
|
||||
for i := range p.clusters {
|
||||
clusterPos[i] = pos
|
||||
pos += uint64(len(p.clusters[i]))
|
||||
}
|
||||
checksumPos := pos
|
||||
|
||||
// 7. Pointer lists.
|
||||
p.urlPtrs = make([]byte, 8*count)
|
||||
for i, e := range ents {
|
||||
binary.LittleEndian.PutUint64(p.urlPtrs[8*i:], e.position)
|
||||
}
|
||||
p.clusterPtrs = make([]byte, 8*len(clusterPos))
|
||||
for i, cp := range clusterPos {
|
||||
binary.LittleEndian.PutUint64(p.clusterPtrs[8*i:], cp)
|
||||
}
|
||||
p.titlePtrs = encodeTitlePtrs(ents)
|
||||
|
||||
// 8. Header.
|
||||
p.hdr = header{
|
||||
uuid: deriveUUID(ents),
|
||||
articleCount: count,
|
||||
clusterCount: uint32(len(p.clusters)),
|
||||
urlPtrPos: urlPtrPos,
|
||||
titlePtrPos: titlePtrPos,
|
||||
clusterPtrPos: clusterPtrPos,
|
||||
mimeListPos: mimeListPos,
|
||||
mainPage: noMainPage,
|
||||
layoutPage: noMainPage,
|
||||
checksumPos: checksumPos,
|
||||
}
|
||||
if w.mainKey != "" {
|
||||
if mi, ok := index[w.mainKey]; ok {
|
||||
p.hdr.mainPage = mi
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// clusterBuf accumulates blobs destined for one cluster.
|
||||
type clusterBuf struct {
|
||||
comp uint8
|
||||
blobs [][]byte
|
||||
size int
|
||||
}
|
||||
|
||||
func (w *Writer) packClusters(ents []*entry) []*clusterBuf {
|
||||
var clusters []*clusterBuf
|
||||
var curText, curBin *clusterBuf
|
||||
|
||||
closeIf := func(c **clusterBuf) {
|
||||
if *c != nil && (*c).size >= maxClusterContent {
|
||||
*c = nil
|
||||
}
|
||||
}
|
||||
add := func(cur **clusterBuf, comp uint8, e *entry) {
|
||||
if *cur == nil {
|
||||
*cur = &clusterBuf{comp: comp}
|
||||
clusters = append(clusters, *cur)
|
||||
}
|
||||
c := *cur
|
||||
e.cluster = uint32(indexOf(clusters, c))
|
||||
e.blob = uint32(len(c.blobs))
|
||||
c.blobs = append(c.blobs, e.data)
|
||||
c.size += len(e.data)
|
||||
}
|
||||
|
||||
for _, e := range ents {
|
||||
if e.redirect {
|
||||
continue
|
||||
}
|
||||
if isTextMime(e.mime) {
|
||||
add(&curText, compZstd, e)
|
||||
closeIf(&curText)
|
||||
} else {
|
||||
add(&curBin, compNone, e)
|
||||
closeIf(&curBin)
|
||||
}
|
||||
}
|
||||
return clusters
|
||||
}
|
||||
|
||||
func indexOf(cs []*clusterBuf, c *clusterBuf) int {
|
||||
for i := range cs {
|
||||
if cs[i] == c {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// encode renders a cluster: an info byte followed by the (optionally zstd)
|
||||
// data section, which is an offset table of (N+1) uint32 values then the N
|
||||
// concatenated blobs. Offsets are relative to the start of the data section.
|
||||
func (c *clusterBuf) encode(noCompress bool) []byte {
|
||||
n := len(c.blobs)
|
||||
tableLen := 4 * (n + 1)
|
||||
total := tableLen
|
||||
for _, b := range c.blobs {
|
||||
total += len(b)
|
||||
}
|
||||
data := make([]byte, tableLen, total)
|
||||
off := uint32(tableLen)
|
||||
binary.LittleEndian.PutUint32(data[0:], off)
|
||||
for i, b := range c.blobs {
|
||||
off += uint32(len(b))
|
||||
binary.LittleEndian.PutUint32(data[4*(i+1):], off)
|
||||
}
|
||||
for _, b := range c.blobs {
|
||||
data = append(data, b...)
|
||||
}
|
||||
|
||||
comp := c.comp
|
||||
if noCompress {
|
||||
comp = compNone
|
||||
}
|
||||
payload := data
|
||||
if comp == compZstd {
|
||||
payload = zstdEncode(data)
|
||||
} else {
|
||||
comp = compNone
|
||||
}
|
||||
out := make([]byte, 0, len(payload)+1)
|
||||
out = append(out, comp) // non-extended: bit 4 clear, uint32 offsets
|
||||
return append(out, payload...)
|
||||
}
|
||||
|
||||
func (e *entry) encodeDirent() []byte {
|
||||
le := binary.LittleEndian
|
||||
var head []byte
|
||||
if e.redirect {
|
||||
head = make([]byte, 12)
|
||||
le.PutUint16(head[0:], redirectEntry)
|
||||
head[3] = e.namespace
|
||||
le.PutUint32(head[8:], e.targetIndex)
|
||||
} else {
|
||||
head = make([]byte, 16)
|
||||
le.PutUint16(head[0:], e.mimeIdx)
|
||||
head[3] = e.namespace
|
||||
le.PutUint32(head[8:], e.cluster)
|
||||
le.PutUint32(head[12:], e.blob)
|
||||
}
|
||||
out := append(head, e.url...)
|
||||
out = append(out, 0)
|
||||
out = append(out, e.title...)
|
||||
return append(out, 0)
|
||||
}
|
||||
|
||||
func encodeMimeList(mimes []string) []byte {
|
||||
var b []byte
|
||||
for _, m := range mimes {
|
||||
b = append(b, m...)
|
||||
b = append(b, 0)
|
||||
}
|
||||
return append(b, 0) // terminating empty string
|
||||
}
|
||||
|
||||
func encodeTitlePtrs(ents []*entry) []byte {
|
||||
order := make([]*entry, len(ents))
|
||||
copy(order, ents)
|
||||
sort.Slice(order, func(i, j int) bool {
|
||||
ti := string(order[i].namespace) + order[i].title
|
||||
tj := string(order[j].namespace) + order[j].title
|
||||
if ti != tj {
|
||||
return ti < tj
|
||||
}
|
||||
return order[i].urlIndex < order[j].urlIndex
|
||||
})
|
||||
b := make([]byte, 4*len(order))
|
||||
for i, e := range order {
|
||||
binary.LittleEndian.PutUint32(b[4*i:], e.urlIndex)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// deriveUUID makes the file deterministic: identical input yields an identical
|
||||
// archive. It hashes every entry's key and content, so repacking the same
|
||||
// mirror is idempotent and diffable.
|
||||
func deriveUUID(ents []*entry) [16]byte {
|
||||
h := md5.New()
|
||||
var n [8]byte
|
||||
for _, e := range ents {
|
||||
h.Write([]byte(key(e.namespace, e.url)))
|
||||
binary.LittleEndian.PutUint64(n[:], uint64(len(e.data)))
|
||||
h.Write(n[:])
|
||||
h.Write(e.data)
|
||||
}
|
||||
var u [16]byte
|
||||
copy(u[:], h.Sum(nil))
|
||||
return u
|
||||
}
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// buildSample writes a small archive exercising text, binary, metadata, a
|
||||
// redirect, and a main page, and returns its bytes.
|
||||
func buildSample(t *testing.T, noCompress bool) []byte {
|
||||
t.Helper()
|
||||
w := NewWriter()
|
||||
w.SetNoCompress(noCompress)
|
||||
w.AddContent(NamespaceContent, "index.html", "Home", "text/html",
|
||||
[]byte("<h1>Home</h1>"+strings.Repeat(" word", 500)))
|
||||
w.AddContent(NamespaceContent, "about/index.html", "About", "text/html",
|
||||
[]byte("<h1>About</h1>"))
|
||||
w.AddContent(NamespaceContent, "_kage/h/logo.png", "", "image/png",
|
||||
[]byte{0x89, 'P', 'N', 'G', 0, 1, 2, 3, 4, 5})
|
||||
w.AddMetadata("Title", "Sample")
|
||||
w.AddMetadata("Language", "eng")
|
||||
w.AddRedirect(NamespaceWellKnown, "mainPage", "Main", NamespaceContent, "index.html")
|
||||
w.SetMainPage(NamespaceContent, "index.html")
|
||||
|
||||
var buf bytes.Buffer
|
||||
n, err := w.WriteTo(&buf)
|
||||
if err != nil {
|
||||
t.Fatalf("WriteTo: %v", err)
|
||||
}
|
||||
if int(n) != buf.Len() {
|
||||
t.Fatalf("WriteTo reported %d bytes, buffer has %d", n, buf.Len())
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestRoundTrip(t *testing.T) {
|
||||
for _, noCompress := range []bool{false, true} {
|
||||
data := buildSample(t, noCompress)
|
||||
r, err := NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
t.Fatalf("NewReader (noCompress=%v): %v", noCompress, err)
|
||||
}
|
||||
|
||||
// Content round-trips with the right mime.
|
||||
home, err := r.Get(NamespaceContent, "index.html")
|
||||
if err != nil {
|
||||
t.Fatalf("Get home: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(string(home.Data), "<h1>Home</h1>") {
|
||||
t.Errorf("home content wrong: %.20q", home.Data)
|
||||
}
|
||||
if home.MimeType != "text/html" {
|
||||
t.Errorf("home mime = %q", home.MimeType)
|
||||
}
|
||||
|
||||
// Binary blob survives byte-for-byte.
|
||||
logo, err := r.Get(NamespaceContent, "_kage/h/logo.png")
|
||||
if err != nil {
|
||||
t.Fatalf("Get logo: %v", err)
|
||||
}
|
||||
if !bytes.Equal(logo.Data, []byte{0x89, 'P', 'N', 'G', 0, 1, 2, 3, 4, 5}) {
|
||||
t.Errorf("logo bytes wrong: %v", logo.Data)
|
||||
}
|
||||
if logo.MimeType != "image/png" {
|
||||
t.Errorf("logo mime = %q", logo.MimeType)
|
||||
}
|
||||
|
||||
// Metadata.
|
||||
meta, err := r.Get(NamespaceMetadata, "Title")
|
||||
if err != nil || string(meta.Data) != "Sample" {
|
||||
t.Errorf("metadata Title = %q, %v", meta.Data, err)
|
||||
}
|
||||
|
||||
// Redirect resolves to the target's content.
|
||||
red, err := r.Get(NamespaceWellKnown, "mainPage")
|
||||
if err != nil {
|
||||
t.Fatalf("Get redirect: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(string(red.Data), "<h1>Home</h1>") {
|
||||
t.Errorf("redirect did not resolve to home: %.20q", red.Data)
|
||||
}
|
||||
|
||||
// Main page.
|
||||
mp, err := r.MainPage()
|
||||
if err != nil {
|
||||
t.Fatalf("MainPage: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(string(mp.Data), "<h1>Home</h1>") {
|
||||
t.Errorf("main page wrong: %.20q", mp.Data)
|
||||
}
|
||||
|
||||
// Misses error.
|
||||
if _, err := r.Get(NamespaceContent, "nope.html"); err == nil {
|
||||
t.Error("expected miss to error")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChecksum(t *testing.T) {
|
||||
data := buildSample(t, false)
|
||||
if len(data) < 16 {
|
||||
t.Fatal("archive too short")
|
||||
}
|
||||
body, sum := data[:len(data)-16], data[len(data)-16:]
|
||||
want := md5.Sum(body)
|
||||
if !bytes.Equal(sum, want[:]) {
|
||||
t.Errorf("trailing MD5 does not match body hash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeterministic(t *testing.T) {
|
||||
a := buildSample(t, false)
|
||||
b := buildSample(t, false)
|
||||
if !bytes.Equal(a, b) {
|
||||
t.Error("same input produced different archives; packing is not deterministic")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMagicAndHeader(t *testing.T) {
|
||||
data := buildSample(t, false)
|
||||
h, err := parseHeader(data[:headerLen])
|
||||
if err != nil {
|
||||
t.Fatalf("parseHeader: %v", err)
|
||||
}
|
||||
if h.checksumPos != uint64(len(data)-16) {
|
||||
t.Errorf("checksumPos = %d, want %d", h.checksumPos, len(data)-16)
|
||||
}
|
||||
if h.articleCount != 6 {
|
||||
t.Errorf("articleCount = %d, want 6", h.articleCount)
|
||||
}
|
||||
if h.mainPage == noMainPage {
|
||||
t.Error("main page not set in header")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user