Tam Nguyen Duc
0849557725
kage parquet export turns a packed archive into a columnar Parquet table, one row per entry, and kage parquet import rebuilds the archive from it. The table follows the open-index/open-markdown field names (doc_id, url, host, crawl_date, content_length, text_length, text) so a kage export sits next to other web-crawl datasets on Hugging Face and reads straight into DuckDB or pandas. Alongside those columns it keeps the raw content bytes and the ZIM structure (namespace, redirect target), so the round trip is lossless: a ZIM exported and reimported is byte-identical. doc_id is a deterministic UUID v5 of the page URL. HTML pages also get a derived plain-text column for full-text search and training use; it plays no part in the round trip, which rebuilds pages from the stored content.
95 lines
2.9 KiB
Go
95 lines
2.9 KiB
Go
package cli
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"github.com/tamnd/kage/dataset"
|
|
)
|
|
|
|
// newParquetCmd groups the two columnar conversions: a ZIM archive out to a
|
|
// Parquet table, and a Parquet table back to a ZIM archive. The table is a flat
|
|
// one-row-per-entry shape ready to publish as a dataset, and the round trip is
|
|
// lossless.
|
|
func newParquetCmd() *cobra.Command {
|
|
cmd := &cobra.Command{
|
|
Use: "parquet",
|
|
Short: "Convert a ZIM archive to a Parquet dataset and back",
|
|
Long: "parquet converts a packed ZIM archive into a columnar Parquet table, one row\n" +
|
|
"per entry with clear columns (url, mime, title, content, extracted text), and\n" +
|
|
"converts such a table back into a ZIM. The table is the shape a dataset host\n" +
|
|
"like Hugging Face expects, and the conversion is lossless: a ZIM round-tripped\n" +
|
|
"through Parquet reproduces every entry, its metadata, and the main page.",
|
|
}
|
|
cmd.AddCommand(newParquetExportCmd())
|
|
cmd.AddCommand(newParquetImportCmd())
|
|
return cmd
|
|
}
|
|
|
|
func newParquetExportCmd() *cobra.Command {
|
|
var out string
|
|
cmd := &cobra.Command{
|
|
Use: "export <file.zim>",
|
|
Short: "Write a Parquet table from a ZIM archive",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
in := args[0]
|
|
if out == "" {
|
|
out = strings.TrimSuffix(in, filepath.Ext(in)) + ".parquet"
|
|
}
|
|
st, err := dataset.ZIMToParquet(in, out, Version)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
printDatasetResult("exported", out)
|
|
printDatasetStats(st, out)
|
|
return nil
|
|
},
|
|
}
|
|
cmd.Flags().StringVarP(&out, "out", "o", "", "output path (default <name>.parquet)")
|
|
return cmd
|
|
}
|
|
|
|
func newParquetImportCmd() *cobra.Command {
|
|
var out string
|
|
cmd := &cobra.Command{
|
|
Use: "import <file.parquet>",
|
|
Short: "Rebuild a ZIM archive from a Parquet table",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
in := args[0]
|
|
if out == "" {
|
|
out = strings.TrimSuffix(in, filepath.Ext(in)) + ".zim"
|
|
}
|
|
st, err := dataset.ParquetToZIM(in, out, Version)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
printDatasetResult("imported", out)
|
|
printDatasetStats(st, out)
|
|
fmt.Fprintf(os.Stderr, " open %s\n", styleAccent.Render("kage open "+out))
|
|
return nil
|
|
},
|
|
}
|
|
cmd.Flags().StringVarP(&out, "out", "o", "", "output path (default <name>.zim)")
|
|
return cmd
|
|
}
|
|
|
|
func printDatasetResult(verb, path string) {
|
|
fmt.Fprintln(os.Stderr, styleOK.Render(verb)+" "+styleTitle.Render(path))
|
|
}
|
|
|
|
func printDatasetStats(st dataset.Stats, path string) {
|
|
fmt.Fprintf(os.Stderr, " %s %d %s %d\n",
|
|
styleAccent.Render("rows"), st.Rows,
|
|
styleAccent.Render("redirects"), st.Redirects)
|
|
fmt.Fprintf(os.Stderr, " %s %s content\n", styleDim.Render("content"), humanBytes(st.ContentBytes))
|
|
if fi, err := os.Stat(path); err == nil {
|
|
fmt.Fprintf(os.Stderr, " %s %s\n", styleAccent.Render("size"), humanBytes(fi.Size()))
|
|
}
|
|
}
|