zzet--gortex/internal/parser/extractor.go

package parser

import (
	"io"

	"github.com/zzet/gortex/internal/graph"
)

// Extractor extracts graph nodes and edges from a single source file.
type Extractor interface {
	Language() string
	Extensions() []string
	Extract(filePath string, src []byte) (*ExtractionResult, error)
}

// PreParser is an optional Extractor capability: a source-rewriting hook run
// before tree-sitter parsing. It lets a language neutralise constructs that
// confuse the grammar (e.g. C-family conditional-compilation directives that
// detach an enclosing declaration) without discarding any code.
//
// Implementations MUST preserve byte offsets and line counts exactly — the
// returned slice has the same length as the input and every newline stays in
// place — so all extracted node ranges, line numbers, and downstream
// resolution remain byte-accurate. Returning nil means "no rewrite".
//
// The hook is a first-class, language-agnostic interface rather than a
// per-language private step: any extractor opts in by implementing it, and the
// same offset-preserving rewrite machinery is then reusable across languages.
type PreParser interface {
	PreParse(src []byte) []byte
}

// ApplyPreParse runs e's PreParse hook when e implements PreParser and the hook
// returns a non-nil rewrite; otherwise it returns src unchanged. The identity
// default means extractors opt in by implementing PreParser, with no behaviour
// change for those that don't.
func ApplyPreParse(e Extractor, src []byte) []byte {
	if pp, ok := e.(PreParser); ok {
		if rewritten := pp.PreParse(src); rewritten != nil {
			return rewritten
		}
	}
	return src
}

// StreamingExtractor is an optional Extractor capability: an extractor that
// reads the file itself — one unit (page / slide / sheet) at a time — through
// an io.ReaderAt, emitting nodes and edges as it goes, instead of receiving the
// whole file as a byte slice. Content extractors (PDF, office documents)
// implement it because their inputs are large and their per-unit processing
// never needs the whole file resident. The indexer prefers this path when it is
// available (on the in-process route), so peak memory is O(one unit), not
// O(file).
//
// emit is called once per produced node with that node's outgoing edges (pass
// nil for none). The extractor MUST recover its own per-document panics or
// return an error; the indexer isolates failures either way.
type StreamingExtractor interface {
	Extractor
	ExtractStream(filePath string, r io.ReaderAt, size int64, emit func(*graph.Node, []*graph.Edge)) error
}

// AssetClass labels a non-code extractor by the kind of large
// binary/document/data artifact it ingests, so the indexer can apply
// corpus-admission caps before a heavy file is ever read and extracted.
// Code extractors return the empty class and are never gated.
type AssetClass string

const (
	// AssetDocument is a human document ingested as searchable content
	// (one KindDoc node per page / slide / sheet): PDF, pptx, xlsx,
	// plain text. Large ones dominate parse memory, so they are subject
	// to a per-file size cap.
	AssetDocument AssetClass = "document"
	// AssetData is a pure binary / columnar / vector artifact recorded
	// as a metadata-only node and never parsed: parquet, npy, lance, …
	// Not code-intelligence content; admitted only when opted in.
	AssetData AssetClass = "data"
	// AssetImage is an image asset (metadata-only node). Cheap to
	// admit; kept un-gated by default.
	AssetImage AssetClass = "image"
)

// AssetExtractor is an optional Extractor capability: a non-code extractor
// that ingests large binary/document/data artifacts rather than source.
// The indexer reads AssetClass at walk time (via the registry, keyed by the
// detected language) to decide admission — a per-file size cap for documents
// and an opt-in gate for data assets — so a content-heavy corpus can't pull
// gigabytes of non-source artifacts into the parse pipeline. Extractors that
// don't implement it are treated as code (always admitted).
type AssetExtractor interface {
	Extractor
	AssetClass() AssetClass
}

// AssetClassOf returns e's AssetClass when e implements AssetExtractor, or the
// empty class (code) otherwise. The single place callers map an extractor to
// its admission class.
func AssetClassOf(e Extractor) AssetClass {
	if a, ok := e.(AssetExtractor); ok {
		return a.AssetClass()
	}
	return ""
}

// ExtractionResult holds the nodes and edges extracted from a single
// file, plus an optional handle to the parse tree the extractor used.
//
// When Tree is non-nil the indexer is responsible for releasing it
// after every per-file consumer (contract extractors, body-fact
// resolvers) has run. Languages whose extractor doesn't have a
// downstream consumer for the tree leave Tree as nil and close their
// own trees internally — the contract pipeline degrades to its regex
// fallback for those languages.
type ExtractionResult struct {
	Nodes []*graph.Node
	Edges []*graph.Edge
	Tree  *ParseTree
	// ConstValues carries the literal value of each KindConstant node
	// whose RHS is a string / numeric literal, for the indexer to persist
	// in the queryable constant_values sidecar (kept out of the gob Meta
	// blob). Keyed by the const node id. Empty for languages / files with
	// no literal constants.
	ConstValues []ConstValue
}

// ConstValue is one constant's persisted literal value: the const node id,
// its file (for file-scoped eviction), and the literal text.
type ConstValue struct {
	NodeID   string
	FilePath string
	Value    string
}