Duc-Tam Nguyen
ffdb4ca969
ZIM is the open single-file archive format Kiwix uses for offline content: a fixed header, a MIME list, URL/title/cluster pointer lists, directory entries, zstd-compressed or stored clusters, and a trailing MD5. The writer lays out a mirror in two passes (assign positions, then emit bytes) and derives the UUID from the content so packing is deterministic. The reader random-accesses entries by namespace and url, follows redirects, and reads xz clusters too so archives from other tooling open. Cross-checked against an independent reader (gozim): header, MIME list, namespaces, urls, dirents, and a non-last cluster's blob all read back byte-for-byte.
49 lines
1.2 KiB
Go
49 lines
1.2 KiB
Go
package zim
|
|
|
|
import (
|
|
"sync"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
// A single shared zstd codec. Both EncodeAll and DecodeAll are safe for
|
|
// concurrent use, so one encoder and one decoder serve the whole process.
|
|
var (
|
|
zstdOnce sync.Once
|
|
zstdEnc *zstd.Encoder
|
|
zstdDec *zstd.Decoder
|
|
)
|
|
|
|
func initZstd() {
|
|
zstdOnce.Do(func() {
|
|
zstdEnc, _ = zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
|
|
zstdDec, _ = zstd.NewReader(nil)
|
|
})
|
|
}
|
|
|
|
func zstdEncode(p []byte) []byte {
|
|
initZstd()
|
|
return zstdEnc.EncodeAll(p, nil)
|
|
}
|
|
|
|
func zstdDecode(p []byte) ([]byte, error) {
|
|
initZstd()
|
|
return zstdDec.DecodeAll(p, nil)
|
|
}
|
|
|
|
// isTextMime reports whether content of this MIME type is worth compressing.
|
|
// Already-compressed media (images, fonts, audio, video, archives) is stored
|
|
// uncompressed so we do not burn CPU inflating it by a few bytes.
|
|
func isTextMime(mime string) bool {
|
|
switch mime {
|
|
case "application/json", "application/xml", "application/javascript",
|
|
"application/x-javascript":
|
|
return true
|
|
}
|
|
if len(mime) >= 5 && mime[:5] == "text/" {
|
|
return true
|
|
}
|
|
// Any structured-XML type: application/rss+xml, image/svg+xml, ...
|
|
return len(mime) >= 4 && mime[len(mime)-4:] == "+xml"
|
|
}
|