zzet--gortex/internal/indexer/shadow_threshold.go

package indexer

import (
	"os"
	"strconv"
	"strings"

	"github.com/zzet/gortex/internal/graph"
)

// defaultShadowMaxFileCount caps the file count above which IndexCtx
// refuses to swap idx.graph for an in-memory shadow during cold start.
// Picked empirically from the in-memory store's prior profiling: at
// ~35k C files (drivers/) the in-memory store peaked at 8.6GB RSS; at
// 60k+ the peak is well past 16GB. The shadow path doubles that
// footprint (in-memory + persisted disk copy at the FlushBulk step),
// so the safe ceiling for a 32GB dev machine sits around 50k source
// files. Above that we fall through to the per-call disk path —
// slower per IndexCtx but bounded RAM.
const defaultShadowMaxFileCount = 50000

// defaultStreamingChunkSize is the per-chunk file count used by the
// streaming-flush path. At ~30 nodes / ~100 edges per file, 5000
// files per chunk yields a ~600MB shadow that fits comfortably in
// RAM even on 8GB build agents.
const defaultStreamingChunkSize = 5000

// mtimeStreamPersistEvery is how many files' mtimes the direct-to-disk
// full-index path buffers before flushing them to the store's FileMtime
// sidecar (see recordStreamedMtime in IndexCtx). It makes a first-ever
// track crash-resumable: a kill after some batches have flushed re-parses
// only the tail on the next boot, instead of re-tracking the whole repo
// from scratch every time it dies under memory pressure. A var, not a
// const, so a test can lower it to observe the incremental flush without
// staging thousands of fixture files.
var mtimeStreamPersistEvery = 500

// shadowMaxFileCount returns the active file-count ceiling for the
// IndexCtx in-memory shadow swap. GORTEX_SHADOW_MAX_FILES overrides
// the default; setting it to 0 disables the shadow entirely (always
// run against the disk store directly), setting it to a high value
// (e.g. 10_000_000) effectively disables the guard. Non-numeric or
// negative values fall back to the default.
func shadowMaxFileCount() int {
	if v := os.Getenv("GORTEX_SHADOW_MAX_FILES"); v != "" {
		n, err := strconv.Atoi(v)
		if err == nil && n >= 0 {
			return n
		}
	}
	return defaultShadowMaxFileCount
}

// defaultShadowMaxBytes caps the total raw input bytes above which the
// in-memory shadow swap is refused even when the file count is under
// shadowMaxFileCount. The file-count ceiling alone is blind to the
// few-huge-files shape (a content repo of a few hundred PDFs / text
// dumps / spreadsheets is well under 50k files but can hold multiple
// GB of bytes that explode into hundreds of thousands of section
// nodes). The all-in-memory shadow holds the whole post-parse graph —
// nodes, edges, retained section text, the in-process search index —
// for the entire pipeline, so a multi-GB input pins multiples of that
// in RAM and OOMs on a content-heavy repo (see #120). 1 GiB of input
// keeps a normal source tree on the fast shadow path while routing
// content-heavy / data-dump repos to the bounded per-call disk path.
const defaultShadowMaxBytes int64 = 1 << 30 // 1 GiB

// shadowMaxBytes returns the active total-input-byte ceiling for the
// in-memory shadow swap. GORTEX_SHADOW_MAX_BYTES overrides the
// default; a high value (e.g. 1<<60) effectively disables the byte
// guard, leaving only the file-count ceiling. Non-numeric, zero, or
// negative values fall back to the default.
func shadowMaxBytes() int64 {
	if v := os.Getenv("GORTEX_SHADOW_MAX_BYTES"); v != "" {
		n, err := strconv.ParseInt(v, 10, 64)
		if err == nil && n > 0 {
			return n
		}
	}
	return defaultShadowMaxBytes
}

// streamingFlushActive reports whether the streaming-flush parse path
// should engage for this IndexCtx. Requirements:
//
//   - the backing store implements graph.BulkLoader (the on-disk backend does)
//   - the file count is above the shadow-max threshold (small repos
//     stay on the all-in-memory shadow path)
//   - GORTEX_STREAMING_FLUSH is enabled (off by default — the
//     streaming path leaves resolve to the disk-only per-call path,
//     so it's only useful when shadow swap can't fit in RAM)
func streamingFlushActive(store graph.Store, fileCount int) bool {
	if _, ok := store.(graph.BulkLoader); !ok {
		return false
	}
	if fileCount <= shadowMaxFileCount() {
		return false
	}
	v := os.Getenv("GORTEX_STREAMING_FLUSH")
	return v == "1" || strings.EqualFold(v, "true")
}

// streamingChunkSize returns the per-chunk file count for the
// streaming-flush path. Override via GORTEX_STREAMING_CHUNK_SIZE.
func streamingChunkSize() int {
	if v := os.Getenv("GORTEX_STREAMING_CHUNK_SIZE"); v != "" {
		n, err := strconv.Atoi(v)
		if err == nil && n > 0 {
			return n
		}
	}
	return defaultStreamingChunkSize
}