Duc-Tam Nguyen
ffdb4ca969
ZIM is the open single-file archive format Kiwix uses for offline content: a fixed header, a MIME list, URL/title/cluster pointer lists, directory entries, zstd-compressed or stored clusters, and a trailing MD5. The writer lays out a mirror in two passes (assign positions, then emit bytes) and derives the UUID from the content so packing is deterministic. The reader random-accesses entries by namespace and url, follows redirects, and reads xz clusters too so archives from other tooling open. Cross-checked against an independent reader (gozim): header, MIME list, namespaces, urls, dirents, and a non-last cluster's blob all read back byte-for-byte.
116 lines
3.7 KiB
Go
116 lines
3.7 KiB
Go
// Package zim reads and writes the ZIM offline-archive format, the open
|
|
// single-file container that Kiwix uses to ship offline content. kage uses it
|
|
// to pack a cloned mirror into one indexable, compressed file that a reader can
|
|
// random-access without unpacking.
|
|
//
|
|
// The package is pure: no network, no clock, no global state beyond a lazily
|
|
// built zstd codec. A ZIM file is laid out as a fixed header, a MIME-type list,
|
|
// three pointer lists (URL, title, cluster), a run of directory entries, a run
|
|
// of clusters that hold the content, and a trailing MD5. Every cross-reference
|
|
// is an absolute file position recorded in the header, so the writer assigns
|
|
// positions in one pass and emits bytes in a second. All integers are
|
|
// little-endian.
|
|
//
|
|
// We write the new namespace scheme (minor version 1): all content lives under
|
|
// the single 'C' namespace, metadata under 'M', and a 'W/mainPage' redirect
|
|
// points at the entry point. Reading handles redirects and both offset widths.
|
|
package zim
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"fmt"
|
|
)
|
|
|
|
// Magic is the ZIM header magic number, the first four bytes of every file.
|
|
const Magic uint32 = 0x44D495A // 72173914
|
|
|
|
const (
|
|
majorVersion uint16 = 5
|
|
minorVersion uint16 = 1 // single 'C' content namespace
|
|
headerLen = 80
|
|
)
|
|
|
|
// Namespaces in the new (minor version 1) scheme.
|
|
const (
|
|
NamespaceContent byte = 'C' // pages and assets
|
|
NamespaceMetadata byte = 'M' // M/Title, M/Date, ...
|
|
NamespaceWellKnown byte = 'W' // W/mainPage redirect
|
|
)
|
|
|
|
// Compression codes carried in the low nibble of a cluster's info byte.
|
|
const (
|
|
compNone uint8 = 1 // stored, no compression
|
|
compXZ uint8 = 4 // xz / LZMA2 (read-only support)
|
|
compZstd uint8 = 5 // zstd (what we write for text)
|
|
|
|
extendedFlag uint8 = 0x10 // bit 4: cluster offsets are uint64, not uint32
|
|
)
|
|
|
|
// Sentinels stored in a directory entry's mimetype field to mark non-content
|
|
// entries. A redirect reuses the cluster slot to hold its target's URL index.
|
|
const (
|
|
redirectEntry uint16 = 0xffff
|
|
linkTargetEntry uint16 = 0xfffe
|
|
deletedEntry uint16 = 0xfffd
|
|
)
|
|
|
|
// noMainPage is the mainPage/layoutPage value meaning "none".
|
|
const noMainPage uint32 = 0xffffffff
|
|
|
|
// header is the 80-byte ZIM header.
|
|
type header struct {
|
|
uuid [16]byte
|
|
articleCount uint32
|
|
clusterCount uint32
|
|
urlPtrPos uint64
|
|
titlePtrPos uint64
|
|
clusterPtrPos uint64
|
|
mimeListPos uint64
|
|
mainPage uint32
|
|
layoutPage uint32
|
|
checksumPos uint64
|
|
}
|
|
|
|
// marshal encodes the header to its 80 wire bytes.
|
|
func (h header) marshal() []byte {
|
|
b := make([]byte, headerLen)
|
|
le := binary.LittleEndian
|
|
le.PutUint32(b[0:], Magic)
|
|
le.PutUint16(b[4:], majorVersion)
|
|
le.PutUint16(b[6:], minorVersion)
|
|
copy(b[8:24], h.uuid[:])
|
|
le.PutUint32(b[24:], h.articleCount)
|
|
le.PutUint32(b[28:], h.clusterCount)
|
|
le.PutUint64(b[32:], h.urlPtrPos)
|
|
le.PutUint64(b[40:], h.titlePtrPos)
|
|
le.PutUint64(b[48:], h.clusterPtrPos)
|
|
le.PutUint64(b[56:], h.mimeListPos)
|
|
le.PutUint32(b[64:], h.mainPage)
|
|
le.PutUint32(b[68:], h.layoutPage)
|
|
le.PutUint64(b[72:], h.checksumPos)
|
|
return b
|
|
}
|
|
|
|
// parseHeader decodes and validates an 80-byte header.
|
|
func parseHeader(b []byte) (header, error) {
|
|
var h header
|
|
if len(b) < headerLen {
|
|
return h, fmt.Errorf("zim: short header: %d bytes", len(b))
|
|
}
|
|
le := binary.LittleEndian
|
|
if le.Uint32(b[0:]) != Magic {
|
|
return h, fmt.Errorf("zim: bad magic, not a ZIM file")
|
|
}
|
|
copy(h.uuid[:], b[8:24])
|
|
h.articleCount = le.Uint32(b[24:])
|
|
h.clusterCount = le.Uint32(b[28:])
|
|
h.urlPtrPos = le.Uint64(b[32:])
|
|
h.titlePtrPos = le.Uint64(b[40:])
|
|
h.clusterPtrPos = le.Uint64(b[48:])
|
|
h.mimeListPos = le.Uint64(b[56:])
|
|
h.mainPage = le.Uint32(b[64:])
|
|
h.layoutPage = le.Uint32(b[68:])
|
|
h.checksumPos = le.Uint64(b[72:])
|
|
return h, nil
|
|
}
|