Tam Nguyen Duc
108b80f2fb
Compressing clusters with zstd is the slow part of packing a large mirror. When a mirror is re-packed after a small change, most clusters are byte-identical and there is no reason to compress them again. pack --incremental keeps a content-addressed cache of compressed clusters in a sidecar next to the output. On the next pack, a cluster whose uncompressed bytes match the cache is served from it instead of being recompressed; only new or changed clusters go through zstd. A cache hit returns exactly what a fresh compression would, so the archive stays deterministic and byte-identical to a cold pack. The zim writer gains a settable cluster compressor so the cache can hook in without changing the format. The cache only writes back the clusters it touched this run, so clusters that left the mirror drop out and it cannot grow without bound.
55 lines
1.5 KiB
Go
55 lines
1.5 KiB
Go
package zim
|
|
|
|
import (
|
|
"sync"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
// A single shared zstd codec. Both EncodeAll and DecodeAll are safe for
|
|
// concurrent use, so one encoder and one decoder serve the whole process.
|
|
var (
|
|
zstdOnce sync.Once
|
|
zstdEnc *zstd.Encoder
|
|
zstdDec *zstd.Decoder
|
|
)
|
|
|
|
func initZstd() {
|
|
zstdOnce.Do(func() {
|
|
zstdEnc, _ = zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
|
|
zstdDec, _ = zstd.NewReader(nil)
|
|
})
|
|
}
|
|
|
|
func zstdEncode(p []byte) []byte {
|
|
initZstd()
|
|
return zstdEnc.EncodeAll(p, nil)
|
|
}
|
|
|
|
// Compress zstd-compresses p with the exact codec the writer uses for its
|
|
// clusters. It is exported so a caller can cache cluster compression across
|
|
// packs and feed the result back through Writer.SetCompress; a cached cluster is
|
|
// then byte-for-byte what a fresh compression would have produced.
|
|
func Compress(p []byte) []byte { return zstdEncode(p) }
|
|
|
|
func zstdDecode(p []byte) ([]byte, error) {
|
|
initZstd()
|
|
return zstdDec.DecodeAll(p, nil)
|
|
}
|
|
|
|
// isTextMime reports whether content of this MIME type is worth compressing.
|
|
// Already-compressed media (images, fonts, audio, video, archives) is stored
|
|
// uncompressed so we do not burn CPU inflating it by a few bytes.
|
|
func isTextMime(mime string) bool {
|
|
switch mime {
|
|
case "application/json", "application/xml", "application/javascript",
|
|
"application/x-javascript":
|
|
return true
|
|
}
|
|
if len(mime) >= 5 && mime[:5] == "text/" {
|
|
return true
|
|
}
|
|
// Any structured-XML type: application/rss+xml, image/svg+xml, ...
|
|
return len(mime) >= 4 && mime[len(mime)-4:] == "+xml"
|
|
}
|