zzet--gortex/internal/indexer/content_admission.go

package indexer

import (
	"context"
	"path/filepath"
	"strings"

	"go.uber.org/zap"

	"github.com/zzet/gortex/internal/gitcmd"
	"github.com/zzet/gortex/internal/graph"
	"github.com/zzet/gortex/internal/parser"
)

// Content-admission skip reasons. They ride on a synthetic file node's
// Meta["skip_reason"] so a dropped asset stays listable and index_health
// rolls it up, mirroring the size / timeout / minified skip telemetry.
const (
	skipReasonLargeDocument  = "large_document"   // document over the per-file cap
	skipReasonVectorData     = "vector_data"      // data asset, data indexing off
	skipReasonLargeData      = "large_data_asset" // data asset over the per-file cap
	skipReasonUntrackedAsset = "untracked_asset"  // asset-class file git does not track
)

// contentAdmissionGate decides, by asset class, whether a non-source artifact
// is admitted into extraction. It is built once before a walk from the
// registry's asset-class map and the resolved config caps, so the hot
// per-file check is a single map lookup. A nil gate is inert — the common
// all-code repo (no asset extractors registered) pays nothing.
type contentAdmissionGate struct {
	classes    map[string]parser.AssetClass
	docLimit   int64
	docCapped  bool
	indexData  bool
	dataLimit  int64
	dataCapped bool
}

// newContentAdmissionGate builds the gate from the indexer's registry and
// config. Returns nil when no asset extractors are registered.
func (idx *Indexer) newContentAdmissionGate() *contentAdmissionGate {
	classes := idx.registry.AssetClasses()
	if len(classes) == 0 {
		return nil
	}
	c := idx.config.Content
	docLimit, docCapped := c.EffectiveMaxDocumentBytes()
	dataLimit, dataCapped := c.EffectiveMaxDataBytes()
	return &contentAdmissionGate{
		classes:    classes,
		docLimit:   docLimit,
		docCapped:  docCapped,
		indexData:  c.IndexData,
		dataLimit:  dataLimit,
		dataCapped: dataCapped,
	}
}

// skip reports whether a file of the given walk-time language and size should
// be dropped before it is read and extracted, and the telemetry reason. A
// non-asset language (the gate has no entry for it) is never gated.
func (g *contentAdmissionGate) skip(lang string, size int64) (string, bool) {
	if g == nil {
		return "", false
	}
	switch g.classes[lang] {
	case parser.AssetDocument:
		if g.docCapped && size > g.docLimit {
			return skipReasonLargeDocument, true
		}
	case parser.AssetData:
		if !g.indexData {
			return skipReasonVectorData, true
		}
		if g.dataCapped && size > g.dataLimit {
			return skipReasonLargeData, true
		}
	}
	return "", false
}

// untrackedAssetGate skips asset-class files (document / data / image) that
// git does not track, when index.skip_untracked_assets is on. It is built
// once per cold walk from the registry's asset-class map and the repo's
// `git ls-files` set. A nil gate is inert — the flag is off, the repo is not
// a git repo, or `git ls-files` failed (admit everything as before).
type untrackedAssetGate struct {
	classes map[string]parser.AssetClass
	tracked map[string]struct{} // absolute paths git tracks
}

// newUntrackedAssetGate builds the gate, returning nil (inert) when the flag
// is off, no asset extractors are registered, or the tracked set can't be
// resolved.
func (idx *Indexer) newUntrackedAssetGate(ctx context.Context, absRoot string) *untrackedAssetGate {
	if !idx.config.SkipUntrackedAssets {
		return nil
	}
	classes := idx.registry.AssetClasses()
	if len(classes) == 0 {
		return nil
	}
	tracked, ok := gitTrackedSet(ctx, absRoot)
	if !ok {
		idx.logger.Info("indexer: skip_untracked_assets is on but the git tracked-set is unavailable; admitting all assets",
			zap.String("root", absRoot))
		return nil
	}
	return &untrackedAssetGate{classes: classes, tracked: tracked}
}

// skip reports whether an asset-class file at absPath should be dropped
// because git does not track it. Non-asset languages (untracked code) and
// tracked assets are never skipped here.
func (g *untrackedAssetGate) skip(lang, absPath string) (string, bool) {
	if g == nil || g.classes[lang] == "" {
		return "", false
	}
	if _, ok := g.tracked[absPath]; ok {
		return "", false
	}
	return skipReasonUntrackedAsset, true
}

// gitTrackedSet returns the set of absolute paths git tracks under root, or
// (nil, false) when root is not a git repo or `git ls-files` fails. The -z
// form is NUL-delimited so paths with spaces / newlines are handled exactly.
func gitTrackedSet(ctx context.Context, root string) (map[string]struct{}, bool) {
	out, err := gitcmd.Output(ctx, root, "ls-files", "-z")
	if err != nil {
		return nil, false
	}
	set := make(map[string]struct{})
	for rel := range strings.SplitSeq(out, "\x00") {
		if rel == "" {
			continue
		}
		set[filepath.Join(root, filepath.FromSlash(rel))] = struct{}{}
	}
	return set, true
}

// contentSkipNode builds a synthetic file node for a content / data asset
// dropped by the admission gate, carrying the skip reason and size so the
// file stays visible (queryable, index_health rollup) without being read or
// parsed — the same treatment size-capped files get.
func contentSkipNode(sf skippedFile) *graph.Node {
	return &graph.Node{
		ID:        sf.relPath,
		Kind:      graph.KindFile,
		Name:      filepath.Base(sf.relPath),
		FilePath:  sf.relPath,
		Language:  sf.lang,
		StartLine: 1,
		Meta: map[string]any{
			"skip_reason":            sf.reason,
			"skipped_due_to_content": true,
			"file_size_bytes":        sf.size,
		},
	}
}

// emitContentSkipNodes adds a synthetic file node for every asset dropped by
// the content-admission gate, so the file stays listable with skip telemetry
// instead of vanishing silently.
func (idx *Indexer) emitContentSkipNodes(skipped []skippedFile) {
	if len(skipped) == 0 {
		return
	}
	nodes := make([]*graph.Node, 0, len(skipped))
	for _, sf := range skipped {
		nodes = append(nodes, contentSkipNode(sf))
	}
	idx.applyRepoPrefix(nodes, nil)
	idx.graph.AddBatch(nodes, nil)
}