zzet--gortex/internal/indexer/content_split.go

package indexer

import (
	"unicode/utf8"

	"go.uber.org/zap"

	"github.com/zzet/gortex/internal/graph"
)

// contentSnippetCap bounds how much of a CONTENT (data_class="content")
// section body is retained on the graph node after its full text has been
// streamed into the dedicated content index. The full body lives in the
// content FTS; the node keeps only this much for display (the why-layer,
// get_symbol_source) — so a repo of hundreds of thousands of sections
// holds ~240 B × N in the graph instead of ~4 KB × N (~17× less text).
const contentSnippetCap = 240

// isContentNode is the indexer-local alias for graph.IsContentNode — the
// shared predicate for a CONTENT section node (KindDoc, data_class=content).
func isContentNode(n *graph.Node) bool {
	return graph.IsContentNode(n)
}

// contentBody returns the full section text carried on a content node, or
// "" if absent.
func contentBody(n *graph.Node) string {
	if n.Meta == nil {
		return ""
	}
	b, _ := n.Meta["section_text"].(string)
	return b
}

// metaInt reads an integer-valued Meta key, tolerating the int / int64 /
// float64 forms a value can take across a gob round-trip.
func metaInt(meta map[string]any, key string) int {
	switch v := meta[key].(type) {
	case int:
		return v
	case int64:
		return int(v)
	case float64:
		return int(v)
	default:
		return 0
	}
}

// contentOrdinal returns the section's ordinal — "ordinal" for text /
// office chunks, "page" for PDF pages.
func contentOrdinal(n *graph.Node) int {
	if o := metaInt(n.Meta, "ordinal"); o != 0 {
		return o
	}
	return metaInt(n.Meta, "page")
}

// safeTruncate returns s clamped to at most maxBytes, never splitting a
// multi-byte rune.
func safeTruncate(s string, maxBytes int) string {
	if len(s) <= maxBytes {
		return s
	}
	cut := maxBytes
	for cut > 0 && !utf8.RuneStart(s[cut]) {
		cut--
	}
	return s[:cut]
}

// leanContentNode strips a content node's full section body down to a
// capped snippet once the full text has been captured for the content
// index. The key stays "section_text" so display consumers keep working;
// only the length changes. "content_indexed" marks that the full body
// lives in the content index (so a reader knows the snippet is partial).
func leanContentNode(n *graph.Node) {
	body := contentBody(n)
	if len(body) <= contentSnippetCap {
		return
	}
	n.Meta["section_text"] = safeTruncate(body, contentSnippetCap)
	n.Meta["content_indexed"] = true
}

// collectContentItems pulls one ContentFTSItem per content section node in
// the batch, carrying the FULL body for the content index. Returns nil
// when the batch has no content.
func collectContentItems(nodes []*graph.Node) []graph.ContentFTSItem {
	var items []graph.ContentFTSItem
	for _, n := range nodes {
		if !isContentNode(n) {
			continue
		}
		body := contentBody(n)
		if body == "" {
			continue
		}
		items = append(items, graph.ContentFTSItem{
			NodeID:   n.ID,
			FilePath: n.FilePath,
			Ordinal:  contentOrdinal(n),
			Body:     body,
		})
	}
	return items
}

// firstContentFilePath returns the FilePath of the first content node in
// the batch (all of one file's content nodes share it), or "" if none. The
// incremental reindex path uses it to wipe a single file's content rows
// before re-streaming.
func firstContentFilePath(nodes []*graph.Node) string {
	for _, n := range nodes {
		if isContentNode(n) {
			return n.FilePath
		}
	}
	return ""
}

// contentSearcher returns the ContentSearcher this index writes content
// section bodies to: the disk sink captured at the shadow swap (so content
// reaches disk even while idx.graph is the in-memory shadow), else
// idx.graph itself when it is a disk store. Returns nil for an in-memory
// store with no content index — in which case content text is left on the
// nodes and falls back to the symbol search (the small-repo / CLI case).
func (idx *Indexer) contentSearcher() graph.ContentSearcher {
	if idx.contentSink != nil {
		return idx.contentSink
	}
	if cs, ok := idx.graph.(graph.ContentSearcher); ok {
		return cs
	}
	return nil
}

// streamContentSections is the per-file content path: it streams a parsed
// file's content section bodies into the dedicated content index, then
// leans the nodes to a snippet — so the bulk text never enters the graph,
// the symbol search, or the materialising code passes. Called in the parse
// worker before AddBatch, so the nodes added to the graph are already
// lean. A nil content searcher (in-memory store) leaves the nodes full.
func (idx *Indexer) streamContentSections(nodes []*graph.Node) {
	cs := idx.contentSearcher()
	if cs == nil {
		return
	}
	items := collectContentItems(nodes)
	if len(items) == 0 {
		return
	}
	if err := cs.AppendContent(idx.RepoPrefix(), items); err != nil {
		// Keep the full body on the nodes if the append failed, so content
		// stays searchable via the symbol-search fallback rather than lost.
		idx.logger.Warn("indexer: content index append failed; leaving section text on nodes",
			zap.Error(err))
		return
	}
	for _, n := range nodes {
		if isContentNode(n) {
			leanContentNode(n)
		}
	}
}