Add a pure-Go zim reader and writer
ZIM is the open single-file archive format Kiwix uses for offline content: a fixed header, a MIME list, URL/title/cluster pointer lists, directory entries, zstd-compressed or stored clusters, and a trailing MD5. The writer lays out a mirror in two passes (assign positions, then emit bytes) and derives the UUID from the content so packing is deterministic. The reader random-accesses entries by namespace and url, follows redirects, and reads xz clusters too so archives from other tooling open. Cross-checked against an independent reader (gozim): header, MIME list, namespaces, urls, dirents, and a non-last cluster's blob all read back byte-for-byte.
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// A single shared zstd codec. Both EncodeAll and DecodeAll are safe for
|
||||
// concurrent use, so one encoder and one decoder serve the whole process.
|
||||
var (
|
||||
zstdOnce sync.Once
|
||||
zstdEnc *zstd.Encoder
|
||||
zstdDec *zstd.Decoder
|
||||
)
|
||||
|
||||
func initZstd() {
|
||||
zstdOnce.Do(func() {
|
||||
zstdEnc, _ = zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
|
||||
zstdDec, _ = zstd.NewReader(nil)
|
||||
})
|
||||
}
|
||||
|
||||
func zstdEncode(p []byte) []byte {
|
||||
initZstd()
|
||||
return zstdEnc.EncodeAll(p, nil)
|
||||
}
|
||||
|
||||
func zstdDecode(p []byte) ([]byte, error) {
|
||||
initZstd()
|
||||
return zstdDec.DecodeAll(p, nil)
|
||||
}
|
||||
|
||||
// isTextMime reports whether content of this MIME type is worth compressing.
|
||||
// Already-compressed media (images, fonts, audio, video, archives) is stored
|
||||
// uncompressed so we do not burn CPU inflating it by a few bytes.
|
||||
func isTextMime(mime string) bool {
|
||||
switch mime {
|
||||
case "application/json", "application/xml", "application/javascript",
|
||||
"application/x-javascript":
|
||||
return true
|
||||
}
|
||||
if len(mime) >= 5 && mime[:5] == "text/" {
|
||||
return true
|
||||
}
|
||||
// Any structured-XML type: application/rss+xml, image/svg+xml, ...
|
||||
return len(mime) >= 4 && mime[len(mime)-4:] == "+xml"
|
||||
}
|
||||
Reference in New Issue
Block a user