From 0849557725643999e691433cd1971690b4e09388 Mon Sep 17 00:00:00 2001 From: Tam Nguyen Duc <1218621+tamnd@users.noreply.github.com> Date: Mon, 15 Jun 2026 19:30:42 +0700 Subject: [PATCH] Add ZIM <-> Parquet conversion for dataset publishing (#23) kage parquet export turns a packed archive into a columnar Parquet table, one row per entry, and kage parquet import rebuilds the archive from it. The table follows the open-index/open-markdown field names (doc_id, url, host, crawl_date, content_length, text_length, text) so a kage export sits next to other web-crawl datasets on Hugging Face and reads straight into DuckDB or pandas. Alongside those columns it keeps the raw content bytes and the ZIM structure (namespace, redirect target), so the round trip is lossless: a ZIM exported and reimported is byte-identical. doc_id is a deterministic UUID v5 of the page URL. HTML pages also get a derived plain-text column for full-text search and training use; it plays no part in the round trip, which rebuilds pages from the stored content. --- CHANGELOG.md | 10 ++ cli/parquet.go | 94 ++++++++++++ cli/root.go | 1 + dataset/dataset.go | 307 ++++++++++++++++++++++++++++++++++++++++ dataset/dataset_test.go | 115 +++++++++++++++ go.mod | 8 ++ go.sum | 30 ++++ zim/reader.go | 61 ++++++++ 8 files changed, 626 insertions(+) create mode 100644 cli/parquet.go create mode 100644 dataset/dataset.go create mode 100644 dataset/dataset_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2eec67b..edcd946 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ All notable changes to kage are recorded here. The format follows ### Added +- `kage parquet export ` and `kage parquet import ` + convert a packed archive to a columnar Parquet table and back. The table is + flat, one row per archive entry, with clear columns (url, host, title, mime, + extracted text, content), so it drops straight into the tooling a dataset host + like Hugging Face expects, and DuckDB or pandas can query it as is. The column + names follow the open-index/open-markdown dataset (`doc_id`, `url`, `host`, + `crawl_date`, `content_length`, `text_length`, `text`), with `doc_id` a + deterministic UUID v5 of the page URL, so a kage export sits alongside other + web-crawl datasets. The conversion is lossless: a ZIM round-tripped through + Parquet reproduces every entry, its metadata, and the main page byte for byte. - `kage pack --incremental` keeps a small cache sidecar next to the output and reuses the compression of any cluster whose bytes have not changed since the last pack. Compressing clusters with zstd is the dominant cost of packing a diff --git a/cli/parquet.go b/cli/parquet.go new file mode 100644 index 0000000..8194078 --- /dev/null +++ b/cli/parquet.go @@ -0,0 +1,94 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + + "github.com/tamnd/kage/dataset" +) + +// newParquetCmd groups the two columnar conversions: a ZIM archive out to a +// Parquet table, and a Parquet table back to a ZIM archive. The table is a flat +// one-row-per-entry shape ready to publish as a dataset, and the round trip is +// lossless. +func newParquetCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "parquet", + Short: "Convert a ZIM archive to a Parquet dataset and back", + Long: "parquet converts a packed ZIM archive into a columnar Parquet table, one row\n" + + "per entry with clear columns (url, mime, title, content, extracted text), and\n" + + "converts such a table back into a ZIM. The table is the shape a dataset host\n" + + "like Hugging Face expects, and the conversion is lossless: a ZIM round-tripped\n" + + "through Parquet reproduces every entry, its metadata, and the main page.", + } + cmd.AddCommand(newParquetExportCmd()) + cmd.AddCommand(newParquetImportCmd()) + return cmd +} + +func newParquetExportCmd() *cobra.Command { + var out string + cmd := &cobra.Command{ + Use: "export ", + Short: "Write a Parquet table from a ZIM archive", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + in := args[0] + if out == "" { + out = strings.TrimSuffix(in, filepath.Ext(in)) + ".parquet" + } + st, err := dataset.ZIMToParquet(in, out, Version) + if err != nil { + return err + } + printDatasetResult("exported", out) + printDatasetStats(st, out) + return nil + }, + } + cmd.Flags().StringVarP(&out, "out", "o", "", "output path (default .parquet)") + return cmd +} + +func newParquetImportCmd() *cobra.Command { + var out string + cmd := &cobra.Command{ + Use: "import ", + Short: "Rebuild a ZIM archive from a Parquet table", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + in := args[0] + if out == "" { + out = strings.TrimSuffix(in, filepath.Ext(in)) + ".zim" + } + st, err := dataset.ParquetToZIM(in, out, Version) + if err != nil { + return err + } + printDatasetResult("imported", out) + printDatasetStats(st, out) + fmt.Fprintf(os.Stderr, " open %s\n", styleAccent.Render("kage open "+out)) + return nil + }, + } + cmd.Flags().StringVarP(&out, "out", "o", "", "output path (default .zim)") + return cmd +} + +func printDatasetResult(verb, path string) { + fmt.Fprintln(os.Stderr, styleOK.Render(verb)+" "+styleTitle.Render(path)) +} + +func printDatasetStats(st dataset.Stats, path string) { + fmt.Fprintf(os.Stderr, " %s %d %s %d\n", + styleAccent.Render("rows"), st.Rows, + styleAccent.Render("redirects"), st.Redirects) + fmt.Fprintf(os.Stderr, " %s %s content\n", styleDim.Render("content"), humanBytes(st.ContentBytes)) + if fi, err := os.Stat(path); err == nil { + fmt.Fprintf(os.Stderr, " %s %s\n", styleAccent.Render("size"), humanBytes(fi.Size())) + } +} diff --git a/cli/root.go b/cli/root.go index 62f2c3e..6fe93ac 100644 --- a/cli/root.go +++ b/cli/root.go @@ -57,6 +57,7 @@ func newRoot() *cobra.Command { root.AddCommand(newServeCmd()) root.AddCommand(newPackCmd()) root.AddCommand(newOpenCmd()) + root.AddCommand(newParquetCmd()) return root } diff --git a/dataset/dataset.go b/dataset/dataset.go new file mode 100644 index 0000000..b412b77 --- /dev/null +++ b/dataset/dataset.go @@ -0,0 +1,307 @@ +// Package dataset converts between a packed ZIM archive and a columnar Parquet +// file. The Parquet form is a flat table with one row per archive entry and +// clear columns (url, mime, title, content, extracted text, redirect target), +// which is the shape a dataset host such as Hugging Face expects. The conversion +// is lossless: every entry, its metadata, and the main page survive a ZIM -> +// Parquet -> ZIM round trip, so the table doubles as an archival representation, +// not just an export. +package dataset + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/google/uuid" + "github.com/parquet-go/parquet-go" + "golang.org/x/net/html" + + "github.com/tamnd/kage/zim" +) + +// Row is one archive entry as a Parquet record. A content or metadata entry +// carries its bytes in Content, its type in Mime, and, for HTML, its visible +// text in Text. A redirect sets IsRedirect and names its destination in +// RedirectTarget ("/", e.g. "C/index.html"), with Content empty. +// +// The leading columns (doc_id, url, host, crawl_date, the length pair, text) +// follow the field names the open-index/open-markdown dataset uses, so a kage +// export drops into the same tooling other web-crawl datasets on Hugging Face +// are read with. The trailing columns (namespace, the redirect pair, content) +// are kage's own: they carry the raw bytes and the ZIM structure that make the +// table a lossless, reversible copy of the archive rather than a one-way export. +// +// Namespace is the single-letter ZIM namespace: "C" for pages and assets, "M" +// for archive metadata (Title, Description, Language, ...), "W" for well-known +// entries such as the main-page pointer. Keeping every namespace as a row is +// what makes the table reversible. +type Row struct { + DocID string `parquet:"doc_id,dict"` + URL string `parquet:"url"` + Host string `parquet:"host,dict"` + Title string `parquet:"title"` + Mime string `parquet:"mime,dict"` + CrawlDate string `parquet:"crawl_date,dict"` + ContentLength int64 `parquet:"content_length"` + TextLength int64 `parquet:"text_length"` + Text string `parquet:"text"` + Namespace string `parquet:"namespace,dict"` + IsRedirect bool `parquet:"is_redirect"` + RedirectTarget string `parquet:"redirect_target"` + Content []byte `parquet:"content"` +} + +// docNamespace is the UUID v5 namespace for kage doc_id values: the standard +// URL namespace, matching how open-markdown derives a deterministic id from a +// page's canonical URL. +var docNamespace = uuid.NameSpaceURL + +// docID returns the deterministic UUID v5 an entry gets in the doc_id column, +// derived from its host and url so the same page always hashes to the same id +// across exports. Entries with no host (metadata, well-known) fall back to the +// namespaced url, which is still stable. +func docID(host, namespace, url string) string { + name := namespace + "/" + url + if host != "" { + name = host + "/" + url + } + return uuid.NewSHA1(docNamespace, []byte(name)).String() +} + +// Stats summarises a conversion for the CLI to report. +type Stats struct { + Rows int64 // total entries written + Redirects int64 // of those, redirects + ContentBytes int64 // sum of stored content bytes (uncompressed) +} + +// writeBatch bounds how many rows are buffered before a write to the Parquet +// writer. It only paces the calls; the writer manages its own row groups. +const writeBatch = 256 + +// ZIMToParquet reads the ZIM at zimPath and writes a Parquet table to outPath, +// one row per entry. The archive's main page is recorded both as its W/mainPage +// redirect row and as file-level metadata, and a short generator/source line is +// attached so the dataset is self-describing. version is kage's version string. +func ZIMToParquet(zimPath, outPath, version string) (Stats, error) { + r, err := zim.Open(zimPath) + if err != nil { + return Stats{}, err + } + defer func() { _ = r.Close() }() + + f, err := os.Create(outPath) + if err != nil { + return Stats{}, err + } + bw := bufio.NewWriter(f) + pw := parquet.NewGenericWriter[Row](bw, parquet.Compression(&parquet.Zstd)) + + host := metaValue(r, "Source") + if host == "" { + host = metaValue(r, "Name") + } + host = strings.ToLower(host) + crawlDate := metaValue(r, "Date") + + if ns, u, ok := r.MainPageRef(); ok { + pw.SetKeyValueMetadata("kage.main_page", string(ns)+"/"+u) + } + pw.SetKeyValueMetadata("kage.generator", strings.TrimSpace("kage "+version)) + pw.SetKeyValueMetadata("kage.source", filepath.Base(zimPath)) + if host != "" { + pw.SetKeyValueMetadata("kage.host", host) + } + + var st Stats + count := r.Count() + batch := make([]Row, 0, writeBatch) + flush := func() error { + if len(batch) == 0 { + return nil + } + if _, err := pw.Write(batch); err != nil { + return err + } + batch = batch[:0] + return nil + } + + for i := uint32(0); i < count; i++ { + e, err := r.EntryAt(i) + if err != nil { + return st, fmt.Errorf("read entry %d: %w", i, err) + } + row := Row{ + Namespace: string(e.Namespace), + URL: e.URL, + Title: e.Title, + Host: host, + CrawlDate: crawlDate, + DocID: docID(host, string(e.Namespace), e.URL), + } + if e.Redirect { + row.IsRedirect = true + row.RedirectTarget = string(e.RedirectNamespace) + "/" + e.RedirectURL + st.Redirects++ + } else { + row.Mime = e.MimeType + row.Content = e.Data + row.ContentLength = int64(len(e.Data)) + st.ContentBytes += int64(len(e.Data)) + if e.MimeType == "text/html" { + row.Text = htmlText(e.Data) + row.TextLength = int64(len(row.Text)) + } + } + st.Rows++ + batch = append(batch, row) + if len(batch) == cap(batch) { + if err := flush(); err != nil { + return st, err + } + } + } + if err := flush(); err != nil { + return st, err + } + if err := pw.Close(); err != nil { + return st, err + } + if err := bw.Flush(); err != nil { + return st, err + } + return st, f.Close() +} + +// ParquetToZIM reads the Parquet table at parquetPath and writes the ZIM archive +// it describes to outPath, reproducing every entry, its metadata, and the main +// page. version is unused for now; it is accepted so the signature matches its +// sibling and can record provenance later. +func ParquetToZIM(parquetPath, outPath, _ string) (Stats, error) { + f, err := os.Open(parquetPath) + if err != nil { + return Stats{}, err + } + defer func() { _ = f.Close() }() + + pr := parquet.NewGenericReader[Row](f) + defer func() { _ = pr.Close() }() + + w := zim.NewWriter() + var st Stats + var mainNS byte + var mainURL string + haveMain := false + + buf := make([]Row, writeBatch) + for { + n, readErr := pr.Read(buf) + for i := 0; i < n; i++ { + row := buf[i] + ns := namespaceByte(row.Namespace) + if row.IsRedirect { + tns, turl := splitTarget(row.RedirectTarget) + w.AddRedirect(ns, row.URL, row.Title, tns, turl) + if ns == zim.NamespaceWellKnown && row.URL == "mainPage" { + mainNS, mainURL, haveMain = tns, turl, true + } + st.Redirects++ + } else { + w.AddContent(ns, row.URL, row.Title, row.Mime, row.Content) + st.ContentBytes += int64(len(row.Content)) + } + st.Rows++ + } + if readErr == io.EOF { + break + } + if readErr != nil { + return st, readErr + } + if n == 0 { + break + } + } + if haveMain { + w.SetMainPage(mainNS, mainURL) + } + + out, err := os.Create(outPath) + if err != nil { + return st, err + } + obw := bufio.NewWriter(out) + if _, err := w.WriteTo(obw); err != nil { + _ = out.Close() + return st, err + } + if err := obw.Flush(); err != nil { + _ = out.Close() + return st, err + } + return st, out.Close() +} + +// metaValue reads an M/ metadata entry as a string, returning "" when it is +// absent. It lets the exporter fill the host and crawl_date columns from the +// archive's own metadata. +func metaValue(r *zim.Reader, name string) string { + b, err := r.Get(zim.NamespaceMetadata, name) + if err != nil { + return "" + } + return string(b.Data) +} + +// namespaceByte turns a one-letter namespace column back into the ZIM byte, +// defaulting to the content namespace for an unexpectedly empty value. +func namespaceByte(s string) byte { + if s == "" { + return zim.NamespaceContent + } + return s[0] +} + +// splitTarget parses a "/" redirect target back into its parts. +// The namespace is a single byte, so the url is everything after the first two +// characters; a malformed value yields the content namespace and the raw string. +func splitTarget(s string) (byte, string) { + if len(s) >= 2 && s[1] == '/' { + return s[0], s[2:] + } + return zim.NamespaceContent, s +} + +// htmlText extracts the visible text of an HTML document: the concatenated text +// nodes outside script, style, and noscript, with runs of whitespace collapsed +// to single spaces. It is a derived convenience column for dataset consumers and +// plays no part in the round trip, which reconstructs pages from Content. +func htmlText(data []byte) string { + doc, err := html.Parse(strings.NewReader(string(data))) + if err != nil { + return "" + } + var b strings.Builder + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode { + switch n.Data { + case "script", "style", "noscript": + return + } + } + if n.Type == html.TextNode { + b.WriteString(n.Data) + b.WriteByte(' ') + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + return strings.Join(strings.Fields(b.String()), " ") +} diff --git a/dataset/dataset_test.go b/dataset/dataset_test.go new file mode 100644 index 0000000..70d7e03 --- /dev/null +++ b/dataset/dataset_test.go @@ -0,0 +1,115 @@ +package dataset + +import ( + "os" + "path/filepath" + "testing" + + "github.com/tamnd/kage/zim" +) + +// buildZIM writes a small archive with a page, an asset, metadata, and a +// main-page redirect to a temp file, and returns its path. +func buildZIM(t *testing.T) string { + t.Helper() + w := zim.NewWriter() + w.AddContent(zim.NamespaceContent, "index.html", "Home", + "text/html", []byte("Home

Hello

World

")) + w.AddContent(zim.NamespaceContent, "logo.png", "", "image/png", []byte{0x89, 'P', 'N', 'G', 1, 2, 3}) + w.AddMetadata("Title", "Test Site") + w.AddMetadata("Language", "eng") + w.SetMainPage(zim.NamespaceContent, "index.html") + w.AddRedirect(zim.NamespaceWellKnown, "mainPage", "", zim.NamespaceContent, "index.html") + + path := filepath.Join(t.TempDir(), "site.zim") + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + if _, err := w.WriteTo(f); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + return path +} + +func TestZIMParquetRoundTrip(t *testing.T) { + src := buildZIM(t) + dir := t.TempDir() + pq := filepath.Join(dir, "site.parquet") + dst := filepath.Join(dir, "site2.zim") + + exp, err := ZIMToParquet(src, pq, "test") + if err != nil { + t.Fatalf("ZIMToParquet: %v", err) + } + if exp.Rows == 0 { + t.Fatal("exported zero rows") + } + if exp.Redirects == 0 { + t.Fatal("expected the mainPage redirect to be exported") + } + + imp, err := ParquetToZIM(pq, dst, "test") + if err != nil { + t.Fatalf("ParquetToZIM: %v", err) + } + if imp.Rows != exp.Rows { + t.Fatalf("row count changed across round trip: exported %d, imported %d", exp.Rows, imp.Rows) + } + if imp.Redirects != exp.Redirects { + t.Fatalf("redirect count changed: exported %d, imported %d", exp.Redirects, imp.Redirects) + } + + // The rebuilt archive must serve the same content and resolve its main page. + r, err := zim.Open(dst) + if err != nil { + t.Fatalf("open rebuilt zim: %v", err) + } + defer func() { _ = r.Close() }() + + home, err := r.Get(zim.NamespaceContent, "index.html") + if err != nil { + t.Fatalf("get index.html: %v", err) + } + if got := string(home.Data); got != "Home

Hello

World

" { + t.Fatalf("page content changed: %q", got) + } + if home.MimeType != "text/html" { + t.Fatalf("page mime changed: %q", home.MimeType) + } + + logo, err := r.Get(zim.NamespaceContent, "logo.png") + if err != nil { + t.Fatalf("get logo.png: %v", err) + } + if string(logo.Data) != string([]byte{0x89, 'P', 'N', 'G', 1, 2, 3}) { + t.Fatal("asset bytes changed across round trip") + } + + title, err := r.Get(zim.NamespaceMetadata, "Title") + if err != nil { + t.Fatalf("get M/Title: %v", err) + } + if string(title.Data) != "Test Site" { + t.Fatalf("metadata changed: %q", string(title.Data)) + } + + main, err := r.MainPage() + if err != nil { + t.Fatalf("main page not set after round trip: %v", err) + } + if main.URL != "index.html" { + t.Fatalf("main page url changed: %q", main.URL) + } +} + +func TestHTMLTextStripsScript(t *testing.T) { + got := htmlText([]byte("

Visible

Text\nhere

")) + want := "Visible Text here" + if got != want { + t.Fatalf("htmlText = %q, want %q", got, want) + } +} diff --git a/go.mod b/go.mod index 11be999..c98cdcf 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,9 @@ require ( github.com/charmbracelet/fang v1.0.0 github.com/go-rod/rod v0.116.2 github.com/go-rod/stealth v0.4.9 + github.com/google/uuid v1.6.0 github.com/klauspost/compress v1.18.6 + github.com/parquet-go/parquet-go v0.30.1 github.com/spf13/cobra v1.10.2 github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6 golang.org/x/image v0.42.0 @@ -15,6 +17,7 @@ require ( ) require ( + github.com/andybalholm/brotli v1.1.1 // indirect github.com/charmbracelet/colorprofile v0.3.3 // indirect github.com/charmbracelet/ultraviolet v0.0.0-20251106190538-99ea45596692 // indirect github.com/charmbracelet/x/ansi v0.11.0 // indirect @@ -33,8 +36,12 @@ require ( github.com/muesli/mango-cobra v1.2.0 // indirect github.com/muesli/mango-pflag v0.1.0 // indirect github.com/muesli/roff v0.1.0 // indirect + github.com/parquet-go/bitpack v1.0.0 // indirect + github.com/parquet-go/jsonlite v1.0.0 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect + github.com/twpayne/go-geom v1.6.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/ysmood/fetchup v0.2.3 // indirect github.com/ysmood/goob v0.4.0 // indirect @@ -44,4 +51,5 @@ require ( golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.38.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/go.sum b/go.sum index e099953..37fd3b7 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,13 @@ charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410 h1:D9PbaszZYpB4nj+d6HTWr1onlmlyuGVNfL9gAi8iB3k= charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410/go.mod h1:1qZyvvVCenJO2M1ac2mX0yyiIZJoZmDM4DG4s0udJkU= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY= +github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= github.com/charmbracelet/colorprofile v0.3.3 h1:DjJzJtLP6/NZ8p7Cgjno0CKGr7wwRJGxWUwh2IyhfAI= @@ -34,6 +42,12 @@ github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA= github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg= github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4= github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= @@ -52,6 +66,14 @@ github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0= github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= +github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA= +github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs= +github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU= +github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0= +github.com/parquet-go/parquet-go v0.30.1 h1:Oy6ganNrAdFiVwy7wNmWagfPTWA2X9Z3tVHBc7JtuX8= +github.com/parquet-go/parquet-go v0.30.1/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -63,10 +85,14 @@ github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4= +github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028= github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6 h1:VQpB2SpK88C6B5lPHTuSZKb2Qee1QWwiFlC5CKY4AW0= github.com/webview/webview_go v0.0.0-20240831120633-6173450d4dd6/go.mod h1:yE65LFCeWf4kyWD5re+h4XNvOHJEXOCOuJZ4v8l5sgk= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns= github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= @@ -97,6 +123,10 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/zim/reader.go b/zim/reader.go index 6d84e74..9601b64 100644 --- a/zim/reader.go +++ b/zim/reader.go @@ -108,6 +108,67 @@ func (r *Reader) MainPage() (Blob, error) { return r.blobAtIndex(r.hdr.mainPage, 0) } +// Entry is one directory entry as stored, returned by EntryAt. A redirect keeps +// Data nil and names its target in RedirectNamespace/RedirectURL; any other +// entry carries its bytes in Data and its type in MimeType. Unlike Get, EntryAt +// does not follow redirects, so a caller can round-trip every entry, the +// redirects included. +type Entry struct { + Namespace byte + URL string + Title string + MimeType string + Redirect bool + RedirectNamespace byte + RedirectURL string + Data []byte +} + +// EntryAt returns the directory entry at idx, where 0 <= idx < Count, in the +// archive's URL order. It is the iteration counterpart to Get: it exposes every +// entry exactly as stored, including metadata and redirects, which is what an +// exporter needs to reproduce the archive. +func (r *Reader) EntryAt(idx uint32) (Entry, error) { + d, err := r.direntAtIndex(idx) + if err != nil { + return Entry{}, err + } + e := Entry{Namespace: d.namespace, URL: d.url, Title: d.title} + if d.redirect { + e.Redirect = true + td, err := r.direntAtIndex(d.targetIndex) + if err != nil { + return Entry{}, fmt.Errorf("zim: redirect target of %c/%s: %w", d.namespace, d.url, err) + } + e.RedirectNamespace = td.namespace + e.RedirectURL = td.url + return e, nil + } + data, err := r.blobData(d.cluster, d.blob) + if err != nil { + return Entry{}, err + } + if int(d.mimeIdx) < len(r.mimes) { + e.MimeType = r.mimes[d.mimeIdx] + } + e.Data = data + return e, nil +} + +// MainPageRef returns the namespace and url of the archive's entry point and +// whether one is set, so an exporter can record which entry is the main page +// without following the W/mainPage redirect. +func (r *Reader) MainPageRef() (byte, string, bool) { + if r.hdr.mainPage == noMainPage { + return 0, "", false + } + d, err := r.direntAtIndex(r.hdr.mainPage) + if err != nil { + return 0, "", false + } + return d.namespace, d.url, true +} + // Get resolves the entry at (namespace, url), following one or more redirects. func (r *Reader) Get(namespace byte, url string) (Blob, error) { target := key(namespace, url)