wehub-resource-sync
a06f331eb8
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
69 lines
2.2 KiB
Go
69 lines
2.2 KiB
Go
package languages
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"io"
|
|
|
|
"github.com/zzet/gortex/internal/graph"
|
|
"github.com/zzet/gortex/internal/parser"
|
|
)
|
|
|
|
// dataAssetShaCap bounds the file size we stream-hash for a data-asset node;
|
|
// larger files (multi-GB datasets) get a size-only node to avoid the full read.
|
|
const dataAssetShaCap = 64 << 20
|
|
|
|
// DataAssetExtractor records pure data / binary assets (columnar stores, array
|
|
// dumps, dataset shards) as a single metadata-only KindFile node — never
|
|
// parsed. They are data, not knowledge: a node keeps them listable and linkable
|
|
// without feeding a binary blob to any grammar. Tagged data_class=data so they
|
|
// stay out of the content retrieval profile (which keys on data_class=content).
|
|
type DataAssetExtractor struct{}
|
|
|
|
func NewDataAssetExtractor() *DataAssetExtractor { return &DataAssetExtractor{} }
|
|
|
|
func (e *DataAssetExtractor) Language() string { return "data" }
|
|
func (e *DataAssetExtractor) Extensions() []string {
|
|
return []string{".parquet", ".npy", ".npz", ".lance", ".arrow", ".feather"}
|
|
}
|
|
func (e *DataAssetExtractor) AssetClass() parser.AssetClass { return parser.AssetData }
|
|
|
|
var (
|
|
_ parser.StreamingExtractor = (*DataAssetExtractor)(nil)
|
|
_ parser.AssetExtractor = (*DataAssetExtractor)(nil)
|
|
)
|
|
|
|
func (e *DataAssetExtractor) Extract(filePath string, src []byte) (*parser.ExtractionResult, error) {
|
|
sum := sha256.Sum256(src)
|
|
return &parser.ExtractionResult{Nodes: []*graph.Node{
|
|
dataAssetNode(filePath, int64(len(src)), hex.EncodeToString(sum[:])),
|
|
}}, nil
|
|
}
|
|
|
|
func (e *DataAssetExtractor) ExtractStream(filePath string, r io.ReaderAt, size int64, emit func(*graph.Node, []*graph.Edge)) error {
|
|
sha := ""
|
|
if size <= dataAssetShaCap {
|
|
h := sha256.New()
|
|
if _, err := io.Copy(h, io.NewSectionReader(r, 0, size)); err == nil {
|
|
sha = hex.EncodeToString(h.Sum(nil))
|
|
}
|
|
}
|
|
emit(dataAssetNode(filePath, size, sha), nil)
|
|
return nil
|
|
}
|
|
|
|
func dataAssetNode(filePath string, size int64, sha string) *graph.Node {
|
|
meta := map[string]any{
|
|
"asset_kind": "data",
|
|
"data_class": "data",
|
|
"size_bytes": int(size),
|
|
}
|
|
if sha != "" {
|
|
meta["sha256"] = sha
|
|
}
|
|
return &graph.Node{
|
|
ID: filePath, Kind: graph.KindFile, Name: filePath,
|
|
FilePath: filePath, StartLine: 1, Language: "data", Meta: meta,
|
|
}
|
|
}
|