chore: import upstream snapshot with attribution
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
// Package serverstack is the single construction path for Gortex's server
|
||||
// stack: it builds graph.Store -> parser.Registry -> indexer -> query.Engine
|
||||
// -> mcp.Server plus every side-effect init, so the daemon, the HTTP
|
||||
// surface, and the one-shot embedded path all share one wiring instead of
|
||||
// three near-identical copies.
|
||||
//
|
||||
// It lives in its own leaf package (not internal/daemon) because the
|
||||
// constructor builds an *mcp.Server, and internal/mcp already imports
|
||||
// internal/daemon one-way; hosting the constructor in internal/daemon
|
||||
// would form the cycle daemon -> mcp -> daemon.
|
||||
package serverstack
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/zzet/gortex/internal/daemon"
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
"github.com/zzet/gortex/internal/platform"
|
||||
)
|
||||
|
||||
// OpenBackend constructs the graph.Store the server will run against,
|
||||
// picking the implementation by name:
|
||||
//
|
||||
// - "" / "memory" — in-process *graph.Graph; nothing persists across
|
||||
// runs; matches every existing test fixture.
|
||||
// - "sqlite" — the pure-Go modernc.org/sqlite store under the resolved
|
||||
// path (defaults to ~/.gortex/store/store.sqlite).
|
||||
//
|
||||
// Returns the store, a cleanup func the caller must defer (closes the
|
||||
// underlying handle on disk-backed stores), and any open error.
|
||||
// allowRebuild permits the on-disk sqlite backend to drop and recreate a
|
||||
// database whose schema version is incompatible. The caller must hold the
|
||||
// store lock; NewSharedServer passes true only in the branch where it acquired
|
||||
// the exclusive flock.
|
||||
func OpenBackend(name, path string, bufferPoolMB uint64, logger *zap.Logger, allowRebuild bool) (graph.Store, func(), error) {
|
||||
switch strings.ToLower(strings.TrimSpace(name)) {
|
||||
case "", "memory", "mem", "in-memory":
|
||||
s := graph.New()
|
||||
return s, func() {}, nil
|
||||
case "sqlite", "sqlite3":
|
||||
resolved, err := resolveBackendPath(path, "store.sqlite")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if logger != nil {
|
||||
logger.Info("opening sqlite backend", zap.String("path", resolved))
|
||||
}
|
||||
return openSqliteBackend(resolved, bufferPoolMB, allowRebuild)
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unknown backend %q (expected: memory, sqlite)", name)
|
||||
}
|
||||
}
|
||||
|
||||
// isSqliteBackend reports whether name selects the on-disk sqlite store.
|
||||
func isSqliteBackend(name string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(name)) {
|
||||
case "sqlite", "sqlite3":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// resolveBackendPath turns an empty backend path into a default under the
|
||||
// unified store directory (~/.gortex/store/<filename>, or the XDG_DATA_HOME
|
||||
// equivalent). Otherwise expands ~ and returns the absolute path, creating
|
||||
// the parent directory so the on-disk store can open the leaf.
|
||||
func resolveBackendPath(in, filename string) (string, error) {
|
||||
in = strings.TrimSpace(in)
|
||||
if in == "" {
|
||||
in = filepath.Join(platform.StoreDir(), filename)
|
||||
} else if strings.HasPrefix(in, "~/") {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve home dir: %w", err)
|
||||
}
|
||||
in = filepath.Join(home, in[2:])
|
||||
}
|
||||
abs, err := filepath.Abs(in)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("abs path %q: %w", in, err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
|
||||
return "", fmt.Errorf("mkdir parent %q: %w", filepath.Dir(abs), err)
|
||||
}
|
||||
return abs, nil
|
||||
}
|
||||
|
||||
// openSqliteBackend opens (or creates) the SQLite store at path. The
|
||||
// pure-Go modernc.org/sqlite driver keeps the binary CGo-free while still
|
||||
// getting a real query planner that drives the graph's secondary indexes.
|
||||
// bufferPoolMB is accepted for signature parity with other on-disk
|
||||
// backends but unused — SQLite sizes its page cache via a pragma.
|
||||
func openSqliteBackend(path string, bufferPoolMB uint64, allowRebuild bool) (graph.Store, func(), error) {
|
||||
_ = bufferPoolMB
|
||||
var opts []store_sqlite.Option
|
||||
if allowRebuild {
|
||||
opts = append(opts, store_sqlite.WithRebuild())
|
||||
}
|
||||
s, err := store_sqlite.Open(path, opts...)
|
||||
if err != nil {
|
||||
hint := "if another gortex daemon is using this store, stop it first (`gortex daemon status` / `gortex daemon stop`)"
|
||||
if pid, ok := daemon.RunningPID(); ok {
|
||||
hint = fmt.Sprintf("a gortex daemon is already running (pid %d) — stop it with `gortex daemon stop`, or use `gortex daemon restart`", pid)
|
||||
}
|
||||
return nil, nil, fmt.Errorf("open sqlite store at %q: %w (%s)", path, err, hint)
|
||||
}
|
||||
return s, func() { _ = s.Close() }, nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package serverstack
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestOpenBackend_MemoryDefault asserts the memory backend (and the empty
|
||||
// default) returns a usable in-process store.
|
||||
func TestOpenBackend_MemoryDefault(t *testing.T) {
|
||||
for _, name := range []string{"", "memory", "mem", "in-memory"} {
|
||||
store, cleanup, err := OpenBackend(name, "", 0, zap.NewNop(), false)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenBackend(%q): %v", name, err)
|
||||
}
|
||||
if store == nil {
|
||||
t.Fatalf("OpenBackend(%q): nil store", name)
|
||||
}
|
||||
if _, ok := store.(*graph.Graph); !ok {
|
||||
t.Errorf("OpenBackend(%q): want *graph.Graph, got %T", name, store)
|
||||
}
|
||||
cleanup()
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenBackend_SqliteOpensFile asserts the sqlite backend opens (and
|
||||
// creates) a store at the resolved path.
|
||||
func TestOpenBackend_SqliteOpensFile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
store, cleanup, err := OpenBackend("sqlite", path, 0, zap.NewNop(), true)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenBackend(sqlite): %v", err)
|
||||
}
|
||||
if store == nil {
|
||||
t.Fatal("nil sqlite store")
|
||||
}
|
||||
cleanup()
|
||||
}
|
||||
|
||||
// TestOpenBackend_Unknown asserts only memory|sqlite are accepted — a
|
||||
// stale backend name (e.g. the removed ladybug) errors rather than
|
||||
// silently falling back.
|
||||
func TestOpenBackend_Unknown(t *testing.T) {
|
||||
if _, _, err := OpenBackend("ladybug", "", 0, zap.NewNop(), false); err == nil {
|
||||
t.Fatal("an unknown backend must error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package serverstack
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/config"
|
||||
"github.com/zzet/gortex/internal/embedding"
|
||||
)
|
||||
|
||||
// EmbedderRequest carries the explicit, per-invocation embedding inputs a
|
||||
// command collected from its flags and environment. ResolveEmbedder
|
||||
// merges these with the on-disk config to decide what provider — if any —
|
||||
// to construct. Fields are exported so cmd entry points can build the
|
||||
// request across the package boundary.
|
||||
type EmbedderRequest struct {
|
||||
// FlagChanged reports whether the `--embeddings` boolean flag was
|
||||
// explicitly set (cmd.Flags().Changed). Only an explicitly-set flag
|
||||
// overrides the config.
|
||||
FlagChanged bool
|
||||
// FlagEnabled is the value of `--embeddings`. Meaningful only when
|
||||
// FlagChanged is true.
|
||||
FlagEnabled bool
|
||||
// FlagURL / FlagModel are `--embeddings-url` / `--embeddings-model`.
|
||||
// A non-empty URL forces the API provider — the most explicit request.
|
||||
FlagURL string
|
||||
FlagModel string
|
||||
}
|
||||
|
||||
// ResolveEmbedder decides which embedding.Provider (if any) to install,
|
||||
// applying a fixed precedence: an explicit URL (flag or env) forces the
|
||||
// API provider; an explicit on/off signal (flag or env) decides
|
||||
// enablement and the provider comes from the `embedding:` config; else
|
||||
// the config decides (default: semantic search ON with the zero-download
|
||||
// static GloVe provider). The returned string describes the decision for
|
||||
// logging ("" when no embedder was built); the SelectionReport records every
|
||||
// backend the local auto-selection tried (empty for other providers) so a
|
||||
// silent degradation to static is observable; a non-nil error means an embedder
|
||||
// was requested but could not be constructed.
|
||||
func ResolveEmbedder(req EmbedderRequest, cfg *config.Config) (embedding.Provider, string, embedding.SelectionReport, error) {
|
||||
if url := firstNonEmpty(req.FlagURL, os.Getenv("GORTEX_EMBEDDINGS_URL")); url != "" {
|
||||
model := firstNonEmpty(req.FlagModel, os.Getenv("GORTEX_EMBEDDINGS_MODEL"))
|
||||
return embedding.NewAPIProvider(url, model), fmt.Sprintf("api (%s)", url), embedding.SelectionReport{}, nil
|
||||
}
|
||||
|
||||
embCfg := config.EmbeddingConfig{}
|
||||
if cfg != nil {
|
||||
embCfg = cfg.Embedding
|
||||
}
|
||||
|
||||
explicitEnabled, haveExplicit := explicitEmbeddingToggle(req)
|
||||
if haveExplicit {
|
||||
if !explicitEnabled {
|
||||
return nil, "", embedding.SelectionReport{}, nil
|
||||
}
|
||||
return buildConfiguredEmbedder(embCfg, "enabled by flag/env")
|
||||
}
|
||||
|
||||
// No explicit flag/env toggle. An explicit `embedding.enabled: false`
|
||||
// still wins and disables the vector channel.
|
||||
if embCfg.Enabled != nil && !*embCfg.Enabled {
|
||||
return nil, "", embedding.SelectionReport{}, nil
|
||||
}
|
||||
// An explicit `embedding.enabled: true` builds whatever provider is
|
||||
// configured, including the baked static GloVe one.
|
||||
if embCfg.Enabled != nil && *embCfg.Enabled {
|
||||
return buildConfiguredEmbedder(embCfg, "enabled by config")
|
||||
}
|
||||
// Otherwise (the default, unset state) build a vector index ONLY when
|
||||
// the user has configured a real, model-backed embedder. The static
|
||||
// GloVe provider's dim-50 word vectors add little over FTS5/BM25 text
|
||||
// search yet cost 0.6-0.7s of every index to build, so it is now
|
||||
// opt-in: FTS5 text search serves the default, and semantic search
|
||||
// turns on when a `local`/`api` provider (or embedding.enabled: true,
|
||||
// or GORTEX_EMBEDDINGS=1) is configured.
|
||||
if isRealEmbedder(embCfg.EmbeddingProviderOrDefault()) {
|
||||
return buildConfiguredEmbedder(embCfg, "real embedder configured")
|
||||
}
|
||||
return nil, "", embedding.SelectionReport{}, nil
|
||||
}
|
||||
|
||||
// isRealEmbedder reports whether the named provider is a model-backed
|
||||
// embedder worth the index-time vector build. The baked `static` GloVe
|
||||
// provider is excluded: its dim-50 averaged word vectors add little over
|
||||
// FTS5 text search, so it no longer builds a vector index by default
|
||||
// (opt in with embedding.enabled: true or GORTEX_EMBEDDINGS=1).
|
||||
func isRealEmbedder(provider string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(provider)) {
|
||||
case "local", "api":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// explicitEmbeddingToggle reports whether the caller gave an explicit
|
||||
// on/off signal for embeddings, and what it was. The flag takes
|
||||
// precedence over GORTEX_EMBEDDINGS.
|
||||
func explicitEmbeddingToggle(req EmbedderRequest) (enabled, haveExplicit bool) {
|
||||
if req.FlagChanged {
|
||||
return req.FlagEnabled, true
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(os.Getenv("GORTEX_EMBEDDINGS"))) {
|
||||
case "1", "true", "yes", "on", "y":
|
||||
return true, true
|
||||
case "0", "false", "no", "off", "n":
|
||||
return false, true
|
||||
default:
|
||||
return false, false
|
||||
}
|
||||
}
|
||||
|
||||
// buildConfiguredEmbedder constructs the provider named by the config
|
||||
// block (defaulting to the static GloVe provider). The returned description
|
||||
// names the backend actually constructed — "local (hugot)" for an
|
||||
// auto-selected local backend, or "local → static fallback" when the chain
|
||||
// degraded — so the startup log tells the truth rather than echoing the
|
||||
// configured name.
|
||||
func buildConfiguredEmbedder(embCfg config.EmbeddingConfig, why string) (embedding.Provider, string, embedding.SelectionReport, error) {
|
||||
provider := embCfg.EmbeddingProviderOrDefault()
|
||||
variant := firstNonEmpty(os.Getenv("GORTEX_EMBEDDINGS_VARIANT"), embCfg.Variant)
|
||||
p, report, err := embedding.NewProviderFromConfigWithReport(embedding.ProviderConfig{
|
||||
Provider: provider,
|
||||
APIURL: embCfg.APIURL,
|
||||
APIModel: embCfg.APIModel,
|
||||
Variant: variant,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, "", report, err
|
||||
}
|
||||
desc := provider
|
||||
switch {
|
||||
case variant != "" && provider == "local":
|
||||
desc = fmt.Sprintf("%s/%s", provider, variant)
|
||||
case provider == "local" && report.Chosen == "static":
|
||||
desc = "local → static fallback"
|
||||
case provider == "local" && report.Chosen != "":
|
||||
desc = fmt.Sprintf("local (%s)", report.Chosen)
|
||||
}
|
||||
return p, fmt.Sprintf("%s — %s", desc, why), report, nil
|
||||
}
|
||||
|
||||
// EmbeddingChunkOptions translates the chunking knobs of an
|
||||
// EmbeddingConfig into the embedding package's ChunkOptions. Zero values
|
||||
// pass through — the chunker substitutes its own defaults.
|
||||
func EmbeddingChunkOptions(cfg *config.Config) embedding.ChunkOptions {
|
||||
if cfg == nil {
|
||||
return embedding.ChunkOptions{}
|
||||
}
|
||||
return embedding.ChunkOptions{
|
||||
ThresholdLines: cfg.Embedding.ChunkThresholdLines,
|
||||
WindowLines: cfg.Embedding.ChunkWindowLines,
|
||||
}
|
||||
}
|
||||
|
||||
// firstNonEmpty returns the first non-empty string argument.
|
||||
func firstNonEmpty(s ...string) string {
|
||||
for _, v := range s {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package serverstack
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/config"
|
||||
"github.com/zzet/gortex/internal/embedding"
|
||||
)
|
||||
|
||||
func boolPtr(b bool) *bool { return &b }
|
||||
|
||||
// clearEmbeddingEnv neutralises the flag/env overrides so a test exercises the
|
||||
// config-driven ResolveEmbedder path deterministically.
|
||||
func clearEmbeddingEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, k := range []string{"GORTEX_EMBEDDINGS", "GORTEX_EMBEDDINGS_URL", "GORTEX_EMBEDDINGS_MODEL", "GORTEX_EMBEDDINGS_VARIANT"} {
|
||||
t.Setenv(k, "")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveEmbedder_StaticDescIsTruthful(t *testing.T) {
|
||||
clearEmbeddingEnv(t)
|
||||
cfg := &config.Config{Embedding: config.EmbeddingConfig{Enabled: boolPtr(true), Provider: "static"}}
|
||||
|
||||
p, desc, report, err := ResolveEmbedder(EmbedderRequest{}, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if p == nil {
|
||||
t.Fatal("expected a static provider")
|
||||
}
|
||||
defer p.Close()
|
||||
if !strings.HasPrefix(desc, "static") {
|
||||
t.Fatalf("desc = %q, want it to start with \"static\"", desc)
|
||||
}
|
||||
if len(report.Attempts) != 0 {
|
||||
t.Fatalf("explicit static should record no selection attempts, got %d", len(report.Attempts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveEmbedder_APIDescIsTruthful(t *testing.T) {
|
||||
clearEmbeddingEnv(t)
|
||||
cfg := &config.Config{Embedding: config.EmbeddingConfig{
|
||||
Enabled: boolPtr(true),
|
||||
Provider: "api",
|
||||
APIURL: "http://localhost:11434",
|
||||
}}
|
||||
|
||||
p, desc, _, err := ResolveEmbedder(EmbedderRequest{}, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if p == nil {
|
||||
t.Fatal("expected an api provider")
|
||||
}
|
||||
defer p.Close()
|
||||
if _, ok := p.(*embedding.APIProvider); !ok {
|
||||
t.Fatalf("expected *embedding.APIProvider, got %T", p)
|
||||
}
|
||||
if !strings.HasPrefix(desc, "api") {
|
||||
t.Fatalf("desc = %q, want it to start with \"api\"", desc)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveEmbedder_LocalDegradesToStatic forces the local auto-selection to
|
||||
// fall all the way through to the static fallback (offline + an empty models
|
||||
// dir so no transformer can load) and asserts the description tells the truth
|
||||
// and the report records the degradation.
|
||||
func TestResolveEmbedder_LocalDegradesToStatic(t *testing.T) {
|
||||
clearEmbeddingEnv(t)
|
||||
t.Setenv("XDG_DATA_HOME", t.TempDir()) // absolute → uncached models dir
|
||||
t.Setenv("GORTEX_EMBEDDING_OFFLINE", "1") // no network download
|
||||
|
||||
cfg := &config.Config{Embedding: config.EmbeddingConfig{Enabled: boolPtr(true), Provider: "local"}}
|
||||
|
||||
p, desc, report, err := ResolveEmbedder(EmbedderRequest{}, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if p == nil {
|
||||
t.Fatal("expected the static fallback provider")
|
||||
}
|
||||
defer p.Close()
|
||||
if report.Chosen != "static" {
|
||||
t.Fatalf("report.Chosen = %q, want \"static\"", report.Chosen)
|
||||
}
|
||||
if len(report.Attempts) == 0 {
|
||||
t.Fatal("expected the failed backend attempts to be recorded")
|
||||
}
|
||||
if !strings.Contains(desc, "static fallback") {
|
||||
t.Fatalf("desc = %q, want it to mention the static fallback", desc)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package serverstack
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/zzet/gortex/internal/config"
|
||||
"github.com/zzet/gortex/internal/resolver"
|
||||
"github.com/zzet/gortex/internal/semantic/lsp"
|
||||
)
|
||||
|
||||
// IsFalsyEnv reports whether the named env var holds a falsy value
|
||||
// (0/false/no/off/n, case-insensitive). Empty/unset is NOT falsy.
|
||||
func IsFalsyEnv(name string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(os.Getenv(name))) {
|
||||
case "0", "false", "no", "off", "n":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// LspDisabledSet builds the set of LSP spec names that should NOT be
|
||||
// auto-registered by Router.RegisterAvailable, merging per-spec config
|
||||
// opt-outs (semantic.providers with enabled:false that resolve to a
|
||||
// known LSP spec) with the comma-separated GORTEX_LSP_DISABLE env var.
|
||||
// The special key "__all__" (from the literal "all"/"*") signals
|
||||
// "skip auto-register everywhere" and is checked separately by callers.
|
||||
func LspDisabledSet(providers []config.SemanticProviderConfig, envVar string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, pc := range providers {
|
||||
if pc.Enabled {
|
||||
continue
|
||||
}
|
||||
if lsp.SpecByName(pc.Name) != nil {
|
||||
out[pc.Name] = true
|
||||
}
|
||||
}
|
||||
for _, raw := range strings.Split(envVar, ",") {
|
||||
name := strings.TrimSpace(raw)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(name, "all") || name == "*" {
|
||||
out["__all__"] = true
|
||||
continue
|
||||
}
|
||||
out[name] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RepoLikelyHasTypeScriptIntent reports whether a repo root carries one
|
||||
// of the canonical TS/JS project markers, used to decide whether to wire
|
||||
// the resolve-time tsserver helper for a tracked repo.
|
||||
func RepoLikelyHasTypeScriptIntent(absRoot string) bool {
|
||||
for _, marker := range []string{"tsconfig.json", "jsconfig.json", "package.json"} {
|
||||
if _, err := os.Stat(filepath.Join(absRoot, marker)); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// RepoLikelyHasPythonIntent reports whether a repo root carries one of the
|
||||
// standard Python project markers, used to decide whether to wire the
|
||||
// resolve-time pyright helper for a tracked repo.
|
||||
func RepoLikelyHasPythonIntent(absRoot string) bool {
|
||||
for _, marker := range []string{
|
||||
"pyproject.toml",
|
||||
"setup.py",
|
||||
"setup.cfg",
|
||||
"requirements.txt",
|
||||
"requirements.in",
|
||||
"Pipfile",
|
||||
"poetry.lock",
|
||||
"uv.lock",
|
||||
"tox.ini",
|
||||
".python-version",
|
||||
} {
|
||||
if _, err := os.Stat(filepath.Join(absRoot, marker)); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if matches, err := filepath.Glob(filepath.Join(absRoot, "*.py")); err == nil && len(matches) > 0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// BuildResolverLSPHelper constructs the resolve-time LSP helper for a
|
||||
// workspace, choosing the router-cached lazy path (poolSize <= 1, reuses
|
||||
// the router's idle reaper) or the fresh-spawn pool path (poolSize > 1,
|
||||
// opt-in via GORTEX_LSP_POOL_SIZE; keeps every spawn alive, so only for
|
||||
// small tracked-workspace counts).
|
||||
func BuildResolverLSPHelper(router *lsp.Router, spec *lsp.ServerSpec, absRoot string, poolSize int, logger *zap.Logger) *lsp.ResolverHelper {
|
||||
if poolSize <= 1 {
|
||||
return lsp.NewLazyResolverHelper(
|
||||
func() (*lsp.Provider, error) {
|
||||
return router.ForSpecWorkspace(spec, absRoot)
|
||||
},
|
||||
absRoot,
|
||||
spec.Extensions,
|
||||
0,
|
||||
logger,
|
||||
)
|
||||
}
|
||||
return lsp.NewPooledResolverHelper(
|
||||
func() (*lsp.Provider, error) {
|
||||
return lsp.SpawnProviderForResolver(spec, absRoot, logger)
|
||||
},
|
||||
absRoot,
|
||||
spec.Extensions,
|
||||
0,
|
||||
poolSize,
|
||||
logger,
|
||||
)
|
||||
}
|
||||
|
||||
// BuildResolverLSPHelperForRepo constructs a per-repo helper that can route
|
||||
// each file extension to the appropriate language server. The returned spec
|
||||
// names are the helpers included in the mux, for logging and diagnostics.
|
||||
func BuildResolverLSPHelperForRepo(router *lsp.Router, absRoot string, poolSize int, logger *zap.Logger) (resolver.LSPHelper, []string) {
|
||||
if router == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var helpers []resolver.LSPHelper
|
||||
var specs []string
|
||||
add := func(name string, enabled bool) {
|
||||
if !enabled {
|
||||
return
|
||||
}
|
||||
spec := lsp.SpecByName(name)
|
||||
if spec == nil || !router.Available(spec) {
|
||||
return
|
||||
}
|
||||
helpers = append(helpers, BuildResolverLSPHelper(router, spec, absRoot, poolSize, logger))
|
||||
specs = append(specs, name)
|
||||
}
|
||||
|
||||
add("typescript-language-server", RepoLikelyHasTypeScriptIntent(absRoot))
|
||||
add("pyright", RepoLikelyHasPythonIntent(absRoot))
|
||||
|
||||
return lsp.NewResolverHelperMux(helpers...), specs
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package serverstack
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRepoLikelyHasPythonIntent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if RepoLikelyHasPythonIntent(dir) {
|
||||
t.Fatalf("empty temp dir should not look like a Python repo")
|
||||
}
|
||||
|
||||
if err := os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !RepoLikelyHasPythonIntent(dir) {
|
||||
t.Fatalf("pyproject.toml should mark a Python repo")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepoLikelyHasPythonIntent_RootScript(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "script.py"), []byte("print('hello')\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !RepoLikelyHasPythonIntent(dir) {
|
||||
t.Fatalf("root-level .py file should mark a Python repo")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
package serverstack
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofrs/flock"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/zzet/gortex/internal/astquery"
|
||||
"github.com/zzet/gortex/internal/config"
|
||||
"github.com/zzet/gortex/internal/contracts"
|
||||
"github.com/zzet/gortex/internal/daemon"
|
||||
"github.com/zzet/gortex/internal/embedding"
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/indexer"
|
||||
gortexmcp "github.com/zzet/gortex/internal/mcp"
|
||||
"github.com/zzet/gortex/internal/parser"
|
||||
"github.com/zzet/gortex/internal/parser/languages"
|
||||
"github.com/zzet/gortex/internal/platform"
|
||||
"github.com/zzet/gortex/internal/query"
|
||||
"github.com/zzet/gortex/internal/savings"
|
||||
"github.com/zzet/gortex/internal/semantic"
|
||||
"github.com/zzet/gortex/internal/semantic/goanalysis"
|
||||
"github.com/zzet/gortex/internal/semantic/lsp"
|
||||
"github.com/zzet/gortex/internal/semantic/scip"
|
||||
"github.com/zzet/gortex/internal/semantic/tstypes"
|
||||
"github.com/zzet/gortex/internal/telemetry"
|
||||
)
|
||||
|
||||
// Lifecycle selects the backend default, whether warm-restart/snapshot
|
||||
// machinery the entry point wires is appropriate, and the store-lock
|
||||
// posture.
|
||||
type Lifecycle int
|
||||
|
||||
const (
|
||||
// LifecycleDaemon is the durable, long-lived daemon: sqlite default,
|
||||
// cross-process store lock.
|
||||
LifecycleDaemon Lifecycle = iota
|
||||
// LifecycleHTTP is the daemon's HTTP surface (gortex daemon --http):
|
||||
// durable, sqlite default, store lock.
|
||||
LifecycleHTTP
|
||||
// LifecycleOneshot is the ephemeral embedded server: memory-only,
|
||||
// FileStore snapshot, no store lock.
|
||||
LifecycleOneshot
|
||||
)
|
||||
|
||||
// Writable reports whether the lifecycle owns a durable on-disk store
|
||||
// (and therefore takes the cross-process store lock).
|
||||
func (l Lifecycle) Writable() bool { return l == LifecycleDaemon || l == LifecycleHTTP }
|
||||
|
||||
// defaultBackend resolves the backend name for an empty cfg.Backend.
|
||||
func (l Lifecycle) defaultBackend() string {
|
||||
if l == LifecycleOneshot {
|
||||
return "memory"
|
||||
}
|
||||
return "sqlite"
|
||||
}
|
||||
|
||||
// SharedServerConfig carries the knobs that vary between the daemon, the
|
||||
// HTTP surface, and the one-shot embedded path. The first block is the
|
||||
// authoritative public surface consumers cite; the rest are
|
||||
// entry-point-resolved options threaded through with the already-loaded
|
||||
// config rather than re-derived here.
|
||||
type SharedServerConfig struct {
|
||||
Lifecycle Lifecycle // backend default + store-lock posture
|
||||
Index string // workspace root the indexer/LSP/bind anchor at
|
||||
Backend string // "" resolves via Lifecycle; else memory|sqlite
|
||||
BackendPath string // "" => ~/.gortex/store/store.sqlite
|
||||
SnapshotPath string // gob+gzip pre-load path; entry point applies it
|
||||
HTTPAddr string // opts the lifecycle into the /mcp HTTP surface
|
||||
Watch bool // filesystem watcher / incremental reindex
|
||||
|
||||
// Entry-point-resolved options (not part of the authoritative surface).
|
||||
Config *config.Config // loaded .gortex.yaml (required)
|
||||
Global *config.GlobalConfig // loaded ~/.gortex/config.yaml
|
||||
Logger *zap.Logger
|
||||
Version string
|
||||
Embedder EmbedderRequest
|
||||
BufferPoolMB uint64
|
||||
SideStores SideStores
|
||||
ScopeWorkspace string
|
||||
ScopeProject string
|
||||
// ActiveProject names the project the MCP server should start scoped
|
||||
// to (multi-repo mode hint). Empty leaves it unset.
|
||||
ActiveProject string
|
||||
// SemanticMode selects the goanalysis provider mode: "callgraph"
|
||||
// builds the call graph; anything else (default) type-checks.
|
||||
SemanticMode string
|
||||
// SavingsPath overrides the token-savings ledger database; empty
|
||||
// defaults to the machine-global sidecar under the data dir
|
||||
// (savings.DefaultDBPath() — the same database the `gortex savings`
|
||||
// CLI reads), independent of SideStores: deriving it from a per-mode
|
||||
// side-store dir would split the ledger between writer and reader.
|
||||
// Tests MUST set it (with SavingsLegacyJSON) to temp paths or the
|
||||
// constructor mutates the developer's real ledger. SavingsRepo
|
||||
// scopes the accumulated totals (empty = workspace-global).
|
||||
// SavingsLegacyJSON names the flat-file era's cumulative
|
||||
// savings.json to import once — its sibling .jsonl event log rides
|
||||
// along; empty uses the historical default location under the
|
||||
// cache dir.
|
||||
SavingsPath string
|
||||
SavingsLegacyJSON string
|
||||
SavingsRepo string
|
||||
}
|
||||
|
||||
// SideStores configures where the agent-authored knowledge stores
|
||||
// persist. Each (dir, repo) pair selects the sidecar DB file (dir, which
|
||||
// opens <dir>/sidecar.sqlite) and the partition key (repo, hashed via
|
||||
// persistence.RepoCacheKey); an empty dir OR repo yields an in-memory
|
||||
// store that never flushes. The zero value therefore makes every store
|
||||
// ephemeral — entry points opt into persistence explicitly, encoding
|
||||
// whether the server owns one repo (per-repo keying) or many (a
|
||||
// workspace-global key).
|
||||
type SideStores struct {
|
||||
// NotesDir / NotesRepo back the durable, repo-partitioned notes and
|
||||
// development-memory stores.
|
||||
NotesDir string
|
||||
NotesRepo string
|
||||
// FeedbackDir / FeedbackRepo back feedback, the combo tracker, and the
|
||||
// frecency tracker (the implicit-signal stores).
|
||||
FeedbackDir string
|
||||
FeedbackRepo string
|
||||
// NotebookPath is the repository-local notebook root (committed to
|
||||
// git so it travels with the repo); empty leaves the notebook
|
||||
// uninitialised.
|
||||
NotebookPath string
|
||||
}
|
||||
|
||||
// SharedServer bundles the constructed stack plus the teardown chain.
|
||||
type SharedServer struct {
|
||||
Graph graph.Store
|
||||
Indexer *indexer.Indexer
|
||||
MultiIndexer *indexer.MultiIndexer // nil in single-repo standalone
|
||||
Engine *query.Engine
|
||||
MCP *gortexmcp.Server
|
||||
ConfigMgr *config.ConfigManager
|
||||
Overlays *daemon.OverlayManager
|
||||
// EmbedderDims is the active embedder's vector dimensionality, or 0
|
||||
// when embeddings are off. The entry point's snapshot warm-start
|
||||
// compares it to a snapshot's vector dims before skipping a re-embed.
|
||||
EmbedderDims int
|
||||
|
||||
// ResolverLSPRegistry / LSPRouter are the resolve-time LSP wiring the
|
||||
// entry point's warmup hooks reference; nil when semantic enrichment
|
||||
// is off.
|
||||
ResolverLSPRegistry *lsp.ResolverHelperRegistry
|
||||
LSPRouter *lsp.Router
|
||||
|
||||
cleanup []func() // run LIFO by Close (backend close, savings flush).
|
||||
}
|
||||
|
||||
// Close runs the teardown chain LIFO.
|
||||
func (s *SharedServer) Close() error {
|
||||
for i := len(s.cleanup) - 1; i >= 0; i-- {
|
||||
s.cleanup[i]()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// telemetrySendInterval bounds how often the daemon flushes the in-memory
|
||||
// usage recorder and attempts to send completed daily aggregates. Long, since
|
||||
// only whole UTC days are ever sent and MaybeSend self-limits to once per day.
|
||||
const telemetrySendInterval = 6 * time.Hour
|
||||
|
||||
// telemetryConsentTTL bounds how often the daemon recorder re-reads consent on
|
||||
// the hot record path — short enough that `gortex telemetry off` stops
|
||||
// recording promptly, long enough to avoid a consent-file read per tool call.
|
||||
const telemetryConsentTTL = 5 * time.Second
|
||||
|
||||
// startTelemetrySender launches the daemon-only periodic flush-and-send loop
|
||||
// and returns a stop function for the cleanup chain. It re-resolves consent
|
||||
// each tick so `gortex telemetry off` stops transmission within one interval,
|
||||
// and is a complete no-op while GORTEX_TELEMETRY_ENDPOINT is unset.
|
||||
func startTelemetrySender(srv *gortexmcp.Server, store *telemetry.Store, dir, version string) func() {
|
||||
sender := telemetry.NewSender(store, dir, version, os.Getenv)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
t := time.NewTicker(telemetrySendInterval)
|
||||
defer t.Stop()
|
||||
flushAndSend := func() {
|
||||
srv.FlushTelemetry()
|
||||
consent := telemetry.ResolveConsent(telemetry.LoadConsentConfig(dir), os.Getenv)
|
||||
sender.MaybeSend(ctx, consent)
|
||||
}
|
||||
flushAndSend() // opportunistic send on startup
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
flushAndSend()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return cancel
|
||||
}
|
||||
|
||||
// NewSharedServer builds the whole server stack — graph.Store ->
|
||||
// parser.Registry -> indexer -> query.Engine -> mcp.Server plus every
|
||||
// side-effect init — through one wiring shared by the daemon, the HTTP
|
||||
// surface, and the one-shot embedded path. Snapshot warm-start and the
|
||||
// per-repo warmup loop stay with the entry point (they orchestrate around
|
||||
// the returned graph/indexer); this constructor is the dedup spine for
|
||||
// the stack wiring itself.
|
||||
func NewSharedServer(cfg SharedServerConfig) (*SharedServer, error) {
|
||||
logger := cfg.Logger
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
conf := cfg.Config
|
||||
if conf == nil {
|
||||
conf = config.Default()
|
||||
}
|
||||
backendName := cfg.Backend
|
||||
if backendName == "" {
|
||||
backendName = cfg.Lifecycle.defaultBackend()
|
||||
}
|
||||
|
||||
// Load user-defined domain-extractor rules (TOML tree-sitter patterns).
|
||||
for _, pattern := range conf.RuleFiles {
|
||||
matches, _ := filepath.Glob(pattern)
|
||||
if len(matches) == 0 {
|
||||
matches = []string{pattern}
|
||||
}
|
||||
for _, rp := range matches {
|
||||
if n, lerr := astquery.LoadUserRulesFile(rp); lerr != nil {
|
||||
logger.Warn("serverstack: failed to load domain rule file",
|
||||
zap.String("file", rp), zap.Error(lerr))
|
||||
} else if n > 0 {
|
||||
logger.Info("serverstack: loaded domain-extractor rules",
|
||||
zap.String("file", rp), zap.Int("rules", n))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
s := &SharedServer{}
|
||||
|
||||
// Cross-process store lock: a writable, on-disk lifecycle
|
||||
// acquires an advisory flock on store.sqlite.lock and fails fast if
|
||||
// another process owns the store. SQLite's in-process writeMu +
|
||||
// busy_timeout serialise in-process writers only; nothing else stops
|
||||
// a second OS process opening the same file and corrupting it.
|
||||
storeLockHeld := false // set true once the exclusive store flock is owned below
|
||||
if cfg.Lifecycle.Writable() && isSqliteBackend(backendName) {
|
||||
storePath, perr := resolveBackendPath(cfg.BackendPath, "store.sqlite")
|
||||
if perr != nil {
|
||||
return nil, perr
|
||||
}
|
||||
lock := flock.New(storePath + ".lock")
|
||||
locked, lerr := lock.TryLock()
|
||||
if lerr != nil {
|
||||
return nil, fmt.Errorf("acquire store lock %q: %w", storePath+".lock", lerr)
|
||||
}
|
||||
if !locked {
|
||||
hint := "the store is already owned by another gortex process"
|
||||
if pid, ok := daemon.RunningPID(); ok {
|
||||
hint = fmt.Sprintf("store already owned by daemon pid %d", pid)
|
||||
}
|
||||
return nil, fmt.Errorf("%s — stop it first (`gortex daemon stop`)", hint)
|
||||
}
|
||||
s.cleanup = append(s.cleanup, func() { _ = lock.Unlock() })
|
||||
storeLockHeld = true
|
||||
}
|
||||
|
||||
// allowRebuild is gated on actually holding the store lock: only then may
|
||||
// the sqlite backend drop and recreate an incompatible-schema DB.
|
||||
g, backendCleanup, err := OpenBackend(backendName, cfg.BackendPath, cfg.BufferPoolMB, logger, storeLockHeld)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.cleanup = append(s.cleanup, backendCleanup)
|
||||
s.Graph = g
|
||||
|
||||
reg := parser.NewRegistry()
|
||||
languages.RegisterAll(reg)
|
||||
languages.RegisterCustomGrammars(reg, conf.Index.Grammars, logger)
|
||||
languages.RegisterExtractorPlugins(reg, conf.Index.ExtractorPlugins, logger)
|
||||
languages.RegisterFallbackChunkers(reg, conf.Index.FallbackChunkers, logger)
|
||||
|
||||
idx := indexer.New(g, reg, conf.Index, logger)
|
||||
s.Indexer = idx
|
||||
|
||||
// Semantic enrichment (opt-in). A daemon-managed LSP router owns
|
||||
// subprocess lifecycle; SCIP + goanalysis providers register eagerly;
|
||||
// Manager.SetLSPRouter installs the bridge so EnrichAll can lazy-spawn
|
||||
// LSPs on demand. RegisterAvailable auto-registers every known LSP
|
||||
// spec whose binary resolves on PATH (per-spec / GORTEX_LSP_DISABLE
|
||||
// opt-out). All lifecycles get this (reconciles the prior mcp-only
|
||||
// drift where the embedded path skipped auto-registration).
|
||||
var semMgr *semantic.Manager
|
||||
if conf.Semantic.Enabled {
|
||||
semCfg := semantic.Config{
|
||||
Enabled: conf.Semantic.Enabled,
|
||||
TimeoutSeconds: conf.Semantic.TimeoutSeconds,
|
||||
EnrichOnWatch: conf.Semantic.EnrichOnWatch,
|
||||
WatchDebounceMs: conf.Semantic.WatchDebounceMs,
|
||||
RefuteUnconfirmed: conf.Semantic.RefuteUnconfirmed,
|
||||
ExcludeGlobs: conf.Semantic.ExcludeGlobs,
|
||||
LSPSweep: conf.Semantic.LSPSweep,
|
||||
EagerLSP: eagerLSPEnabled(conf.Semantic),
|
||||
}
|
||||
for _, pc := range conf.Semantic.Providers {
|
||||
out := semantic.ProviderConfig{
|
||||
Name: pc.Name,
|
||||
Languages: pc.Languages,
|
||||
Command: pc.Command,
|
||||
Args: pc.Args,
|
||||
Env: pc.Env,
|
||||
Mode: pc.Mode,
|
||||
Daemon: pc.Daemon,
|
||||
Priority: pc.Priority,
|
||||
Enabled: pc.Enabled,
|
||||
}
|
||||
if pc.Connect != nil {
|
||||
out.Connect = &semantic.ConnectConfig{
|
||||
Network: pc.Connect.Network,
|
||||
Address: pc.Connect.Address,
|
||||
FallbackSpawn: pc.Connect.FallbackSpawn,
|
||||
}
|
||||
}
|
||||
semCfg.Providers = append(semCfg.Providers, out)
|
||||
}
|
||||
semMgr = semantic.NewManager(semCfg, logger)
|
||||
|
||||
goMode := goanalysis.ModeTypeCheck
|
||||
if cfg.SemanticMode == "callgraph" {
|
||||
goMode = goanalysis.ModeCallGraph
|
||||
}
|
||||
goProvider := goanalysis.NewProvider(goMode, goTypesIncludeTests(), logger)
|
||||
// The go/packages "go-types" provider owns Go enrichment when the
|
||||
// toolchain is present: it type-checks the module once in-process and
|
||||
// stamps every symbol's exact type plus resolves stdlib/dep calls to
|
||||
// real nodes, in a fraction of the time the per-request gopls LSP pass
|
||||
// takes (which additionally leaves external calls as dead stubs). As an
|
||||
// eager provider it claims the "go" language slot, so the LSP router's
|
||||
// gopls spec is skipped as a gap-filler and never spawns. Registration
|
||||
// is gated on Available() (a `go` toolchain on PATH): without it the
|
||||
// slot stays open and gopls / the tree-sitter floor serve Go instead.
|
||||
if goTypesEnrichEnabled(conf.Semantic) && goProvider.Available() {
|
||||
semMgr.RegisterProvider(goProvider)
|
||||
}
|
||||
contracts.SetBindingResolver(goProvider)
|
||||
|
||||
// In-process tree-sitter type resolvers — always-available
|
||||
// (no external binary), supplemental (they coexist with LSP /
|
||||
// SCIP providers instead of competing for the language slot).
|
||||
// Disable one via a `semantic.providers` entry with
|
||||
// `enabled: false` under its name (e.g. "java-types").
|
||||
for _, tp := range tstypes.DefaultProviders(logger) {
|
||||
semMgr.RegisterProvider(tp)
|
||||
}
|
||||
|
||||
lspWorkspace := cfg.Index
|
||||
if lspWorkspace == "" {
|
||||
lspWorkspace, _ = os.Getwd()
|
||||
}
|
||||
lspRouter := lsp.NewRouter(lspWorkspace, logger).
|
||||
WithIdleTimeout(10 * time.Minute).
|
||||
WithReaperInterval(time.Minute).
|
||||
WithMaxAlive(6).
|
||||
WithAdditionalWorkspaceFolders(conf.Semantic.AdditionalWorkspaceFolders).
|
||||
WithEnrichExcludeGlobs(conf.Semantic.ExcludeGlobs).
|
||||
WithEnrichSweepMode(semCfg.LSPSweep)
|
||||
semMgr.SetLSPRouter(lspRouter)
|
||||
|
||||
for _, pc := range semCfg.Providers {
|
||||
if !pc.Enabled {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(pc.Name, "scip-") && pc.Command != "":
|
||||
scipProv := scip.NewProvider(pc.Command, pc.Args, pc.Languages, semCfg.TimeoutSeconds, logger)
|
||||
if pc.Mode == "definitions" {
|
||||
scipProv = scipProv.WithDefinitionsOnly()
|
||||
}
|
||||
semMgr.RegisterProvider(scipProv)
|
||||
case lsp.SpecByName(pc.Name) != nil:
|
||||
var connect *lsp.ConnectSpec
|
||||
if pc.Connect != nil {
|
||||
connect = &lsp.ConnectSpec{
|
||||
Network: pc.Connect.Network,
|
||||
Address: pc.Connect.Address,
|
||||
FallbackSpawn: pc.Connect.FallbackSpawn,
|
||||
}
|
||||
}
|
||||
lspRouter.RegisterSpec(lsp.SpecWithOverridesConnect(
|
||||
lsp.SpecByName(pc.Name), pc.Command, pc.Args, pc.Env, connect))
|
||||
case pc.Daemon:
|
||||
semMgr.RegisterProvider(lsp.NewProvider(pc.Command, pc.Args, pc.Languages, pc.Daemon, pc.MaxParallel, logger))
|
||||
}
|
||||
}
|
||||
|
||||
disabled := LspDisabledSet(conf.Semantic.Providers, os.Getenv("GORTEX_LSP_DISABLE"))
|
||||
var autoRegistered []string
|
||||
if !disabled["__all__"] {
|
||||
autoRegistered = lspRouter.RegisterAvailable(disabled)
|
||||
}
|
||||
|
||||
idx.SetSemanticManager(semMgr)
|
||||
|
||||
if !IsFalsyEnv("GORTEX_LSP_RESOLVER") {
|
||||
s.ResolverLSPRegistry = lsp.NewResolverHelperRegistry()
|
||||
s.LSPRouter = lspRouter
|
||||
idx.SetResolverLSPHelper(s.ResolverLSPRegistry)
|
||||
// Single-repo standalone mode: anchor a "" prefix helper at
|
||||
// the workspace the server points at. Only fires when Index is
|
||||
// set (the embedded path); the daemon leaves Index empty and
|
||||
// registers per-repo helpers via the OnRepoTracked hook.
|
||||
if cfg.Index != "" {
|
||||
if abs, aerr := filepath.Abs(cfg.Index); aerr == nil {
|
||||
helper, specs := BuildResolverLSPHelperForRepo(lspRouter, abs, lsp.ResolverPoolSizeFromEnv(1), logger)
|
||||
if helper != nil {
|
||||
s.ResolverLSPRegistry.Register("", helper)
|
||||
logger.Info("serverstack: single-repo resolve-time LSP helpers registered",
|
||||
zap.Strings("specs", specs))
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.Info("serverstack: resolve-time LSP hot path enabled")
|
||||
} else {
|
||||
logger.Info("serverstack: resolve-time LSP hot path disabled (GORTEX_LSP_RESOLVER=0)")
|
||||
}
|
||||
|
||||
logger.Info("serverstack: semantic enrichment enabled",
|
||||
zap.Int("providers", len(semCfg.Providers)),
|
||||
zap.Strings("lsp_auto_registered", autoRegistered))
|
||||
}
|
||||
|
||||
// Layer the global ~/.gortex/config.yaml `embedding:` block under the
|
||||
// repo-local one before resolving the embedder, so an `embedding:` block in
|
||||
// the global file is honoured while any repo-local non-zero field still
|
||||
// wins. Done here (not at the LLM merge below) so it precedes ResolveEmbedder
|
||||
// and leaves its flag/env precedence untouched. Warn about any unrecognised
|
||||
// top-level keys — most often an `embedding:` block written in the wrong file.
|
||||
embGlobal := cfg.Global
|
||||
if embGlobal == nil {
|
||||
var gerr error
|
||||
if embGlobal, gerr = config.LoadGlobal(); gerr != nil {
|
||||
// A malformed global file is otherwise silently ignored — exactly
|
||||
// when its embedding/llm block would have taken effect.
|
||||
logger.Warn("serverstack: global config could not be parsed — its embedding/llm block is ignored",
|
||||
zap.Error(gerr))
|
||||
}
|
||||
}
|
||||
conf.Embedding = embGlobal.MergeEmbeddingInto(conf.Embedding)
|
||||
// Diagnose unknown keys in the file that was actually loaded (which may be an
|
||||
// overridden path from cfg.Global), not always the default location.
|
||||
globalPath := ""
|
||||
if embGlobal != nil {
|
||||
globalPath = embGlobal.ConfigPath()
|
||||
}
|
||||
if unknown := config.UnknownGlobalKeys(globalPath); len(unknown) > 0 {
|
||||
logger.Warn("serverstack: ~/.gortex/config.yaml contains keys gortex does not recognize",
|
||||
zap.Strings("keys", unknown),
|
||||
zap.String("hint", "see docs/semantic-search.md for embedding config placement"))
|
||||
}
|
||||
|
||||
// Embeddings: explicit flag/env > `embedding:` config > default (on,
|
||||
// static GloVe).
|
||||
embedder, embDesc, embReport, embErr := ResolveEmbedder(cfg.Embedder, conf)
|
||||
// Probe API-backed providers up front so Dimensions() is truthful before
|
||||
// we log it and — crucially — before EmbedderDims gates snapshot-vector
|
||||
// reload. An APIProvider reports 0 until its first embed; without this the
|
||||
// log reads dim:0 and the warm-restart vector reload rejects a
|
||||
// correctly-sized cached index, re-embedding the whole graph needlessly.
|
||||
// Static / local providers know their width natively and don't implement
|
||||
// the prober, so they skip this. Best-effort: a probe failure (bad key /
|
||||
// unreachable URL) only warns — indexing still falls back to BM25.
|
||||
if embedder != nil {
|
||||
if prober, ok := embedder.(interface {
|
||||
ProbeDimensions(context.Context) (int, error)
|
||||
}); ok {
|
||||
if dim, perr := prober.ProbeDimensions(context.Background()); perr != nil {
|
||||
logger.Warn("serverstack: embedding dimension probe failed — width unknown until first embed",
|
||||
zap.Error(perr))
|
||||
} else {
|
||||
logger.Info("serverstack: embedding dimension probed", zap.Int("dim", dim))
|
||||
}
|
||||
}
|
||||
}
|
||||
// Surface every backend the local auto-selection tried. Warn per backend
|
||||
// only when the outcome was actually degraded (fell to the static fallback,
|
||||
// or built nothing) AND the backend could really have worked — a backend
|
||||
// that is simply not compiled into this build (the onnx/gomlx stubs) is
|
||||
// benign noise and stays at debug even on a degradation.
|
||||
outcomeDegraded := (embedder == nil || embReport.Chosen == "static") && len(embReport.Attempts) > 0
|
||||
for _, a := range embReport.Attempts {
|
||||
if outcomeDegraded && !errors.Is(a.Err, embedding.ErrBackendNotCompiled) {
|
||||
logger.Warn("serverstack: embedding backend unavailable — degraded to static fallback",
|
||||
zap.String("backend", a.Backend), zap.Error(a.Err))
|
||||
} else {
|
||||
logger.Debug("serverstack: embedding backend not available",
|
||||
zap.String("backend", a.Backend), zap.Error(a.Err))
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case embErr != nil:
|
||||
logger.Warn("serverstack: embeddings requested but unavailable", zap.Error(embErr))
|
||||
case embedder != nil:
|
||||
logger.Info("serverstack: embeddings enabled",
|
||||
zap.String("provider", embDesc),
|
||||
zap.Int("dim", embedder.Dimensions()))
|
||||
default:
|
||||
logger.Info("serverstack: embeddings disabled")
|
||||
}
|
||||
if embedder != nil {
|
||||
idx.SetEmbedder(embedder)
|
||||
idx.SetEmbeddingChunkOptions(EmbeddingChunkOptions(conf))
|
||||
idx.SetEmbeddingMaxSymbols(conf.Embedding.MaxSymbols)
|
||||
idx.SetEmbeddingAPIConcurrency(conf.Embedding.APIConcurrency)
|
||||
s.EmbedderDims = embedder.Dimensions()
|
||||
}
|
||||
|
||||
cm, err := config.NewConfigManager("")
|
||||
if err != nil {
|
||||
logger.Warn("serverstack: could not load global config", zap.Error(err))
|
||||
}
|
||||
s.ConfigMgr = cm
|
||||
|
||||
var mi *indexer.MultiIndexer
|
||||
if cm != nil {
|
||||
mi = indexer.NewMultiIndexer(g, reg, idx.Search(), cm, logger)
|
||||
if embedder != nil {
|
||||
mi.SetEmbedder(embedder)
|
||||
mi.SetEmbeddingChunkOptions(EmbeddingChunkOptions(conf))
|
||||
mi.SetEmbeddingMaxSymbols(conf.Embedding.MaxSymbols)
|
||||
mi.SetEmbeddingAPIConcurrency(conf.Embedding.APIConcurrency)
|
||||
}
|
||||
// Propagate the semantic manager so per-repo Indexers
|
||||
// created by the MultiIndexer can run LSP enrichment.
|
||||
// Without this, daemon-mode enrichment produces zero
|
||||
// results (enrich:0 in DEFERRED-TIMING).
|
||||
if semMgr != nil {
|
||||
mi.SetSemanticManager(semMgr)
|
||||
}
|
||||
if s.ResolverLSPRegistry != nil {
|
||||
mi.SetResolverLSPHelper(s.ResolverLSPRegistry)
|
||||
if s.LSPRouter != nil {
|
||||
routerRef := s.LSPRouter
|
||||
registryRef := s.ResolverLSPRegistry
|
||||
poolSize := lsp.ResolverPoolSizeFromEnv(1)
|
||||
mi.SetOnRepoTracked(func(prefix, absPath string) {
|
||||
helper, _ := BuildResolverLSPHelperForRepo(routerRef, absPath, poolSize, logger)
|
||||
if helper != nil {
|
||||
registryRef.Register(prefix, helper)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
s.MultiIndexer = mi
|
||||
|
||||
toolPolicyCfg := gortexmcp.ToolPolicyConfig{
|
||||
Preset: conf.MCP.Tools.Preset,
|
||||
Mode: conf.MCP.Tools.Mode,
|
||||
Allow: conf.MCP.Tools.Allow,
|
||||
Deny: conf.MCP.Tools.Deny,
|
||||
}
|
||||
scopeIntentDefaults := conf.Scope.IntentDefaults
|
||||
multiOpts := []gortexmcp.MultiRepoOptions{{
|
||||
MultiIndexer: mi,
|
||||
ConfigManager: cm,
|
||||
ActiveProject: cfg.ActiveProject,
|
||||
ScopeWorkspace: cfg.ScopeWorkspace,
|
||||
ScopeProject: cfg.ScopeProject,
|
||||
ToolPolicy: &toolPolicyCfg,
|
||||
ScopeIntentDefaults: &scopeIntentDefaults,
|
||||
}}
|
||||
|
||||
eng := query.NewEngine(g)
|
||||
eng.SetSearchProvider(idx.Search)
|
||||
eng.ApplyRerankWeights(conf.Search.Weights)
|
||||
s.Engine = eng
|
||||
|
||||
gortexmcp.Version = cfg.Version
|
||||
srv := gortexmcp.NewServer(eng, g, idx, nil, logger, conf.Guards.Rules, multiOpts...)
|
||||
srv.SetArchitecture(conf.Architecture)
|
||||
srv.SetEventRules(conf.Events.Rules)
|
||||
srv.SetArtifacts(conf.Artifacts)
|
||||
srv.SetNamedQueries(conf.Queries)
|
||||
srv.SetSearchConfig(conf.Search)
|
||||
s.MCP = srv
|
||||
|
||||
overlays := daemon.NewOverlayManager(daemon.OverlayIdleTTLFromEnv(0))
|
||||
srv.SetOverlayManager(overlays)
|
||||
s.Overlays = overlays
|
||||
stopOverlayJanitor := overlays.StartJanitor(0, func(dropped int) {
|
||||
logger.Info("overlay janitor: swept idle sessions", zap.Int("dropped", dropped))
|
||||
})
|
||||
s.cleanup = append(s.cleanup, stopOverlayJanitor)
|
||||
|
||||
if semMgr := idx.SemanticManager(); semMgr != nil {
|
||||
srv.SetSemanticManager(semMgr)
|
||||
srv.SetLSPDiagnosticsBroadcasting()
|
||||
}
|
||||
|
||||
// Opt-in usage telemetry. Every entry point records (consent-gated and
|
||||
// fail-silent — nothing happens unless the user enabled it); the daemon
|
||||
// additionally flushes and sends completed daily aggregates on a long
|
||||
// interval, dormant until GORTEX_TELEMETRY_ENDPOINT is configured. Both the
|
||||
// record path (via a TTL-cached resolver) and the send path re-resolve
|
||||
// consent live, so `gortex telemetry off` stops recording within
|
||||
// telemetryConsentTTL and stops transmission within one send interval —
|
||||
// rather than freezing the decision at startup.
|
||||
teleDir := platform.TelemetryDir()
|
||||
teleStore := telemetry.NewStore(teleDir)
|
||||
teleRec := telemetry.NewRecorderFunc(
|
||||
telemetry.CachedConsentResolver(teleDir, telemetryConsentTTL), teleStore)
|
||||
srv.SetTelemetryRecorder(teleRec)
|
||||
s.cleanup = append(s.cleanup, func() { srv.FlushTelemetry() })
|
||||
if cfg.Lifecycle == LifecycleDaemon {
|
||||
// One daemon-session event per process start, dimensioned by backend.
|
||||
// Opt-in + fail-silent: a disabled recorder drops it.
|
||||
telemetry.RecordDaemonSession(teleRec, backendName)
|
||||
s.cleanup = append(s.cleanup, startTelemetrySender(srv, teleStore, teleDir, cfg.Version))
|
||||
}
|
||||
|
||||
// Side-stores. The HTTP path previously wired NONE of these; routing it
|
||||
// through the shared constructor adds notes/memories/savings to it.
|
||||
sideCfg := cfg.SideStores
|
||||
srv.InitFeedback(sideCfg.FeedbackDir, sideCfg.FeedbackRepo)
|
||||
srv.InitNotes(sideCfg.NotesDir, sideCfg.NotesRepo)
|
||||
// Learned tool surface: re-promote tools this workspace promoted in
|
||||
// prior sessions (demoting the ones that fell out of use). Runs after
|
||||
// the NewServer register sweep, so the cold tools are already deferred.
|
||||
srv.InitLearnedTools(sideCfg.NotesDir, sideCfg.NotesRepo)
|
||||
srv.InitMemories(sideCfg.NotesDir, sideCfg.NotesRepo)
|
||||
srv.InitSuppressions(sideCfg.NotesDir, sideCfg.NotesRepo)
|
||||
srv.InitNotebook(sideCfg.NotebookPath)
|
||||
srv.InitCombo(sideCfg.FeedbackDir, sideCfg.FeedbackRepo, gortexmcp.ModeAI)
|
||||
srv.InitFrecency(sideCfg.FeedbackDir, sideCfg.FeedbackRepo, gortexmcp.ModeAI)
|
||||
|
||||
// The savings ledger is machine-global: every entry point defaults to
|
||||
// the same sidecar database the `gortex savings` CLI reads. Deriving
|
||||
// it from a per-mode side-store dir would split the ledger between
|
||||
// writer and reader — the failure mode the flat files had.
|
||||
savingsPath := cfg.SavingsPath
|
||||
if savingsPath == "" {
|
||||
savingsPath = savings.DefaultDBPath()
|
||||
}
|
||||
if savingsStore, err := savings.Open(savingsPath); err == nil {
|
||||
legacyJSON := cfg.SavingsLegacyJSON
|
||||
if legacyJSON == "" {
|
||||
legacyJSON = savings.DefaultPath()
|
||||
}
|
||||
if ierr := savingsStore.ImportLegacy(legacyJSON); ierr != nil {
|
||||
logger.Warn("serverstack: legacy savings import failed", zap.Error(ierr))
|
||||
}
|
||||
srv.InitSavings(savingsStore, cfg.SavingsRepo)
|
||||
s.cleanup = append(s.cleanup, func() { _ = srv.FlushSavings() })
|
||||
} else {
|
||||
logger.Warn("serverstack: savings persistence disabled", zap.Error(err))
|
||||
}
|
||||
|
||||
global := cfg.Global
|
||||
if global == nil {
|
||||
global, _ = config.LoadGlobal()
|
||||
}
|
||||
if global != nil {
|
||||
srv.SetupLLM(global.MergeLLMInto(conf.LLM))
|
||||
// SetupLLM constructs the LLM service lazily and attaches it; nothing
|
||||
// else frees it, so a graceful shutdown would leave a loaded local
|
||||
// model resident (only the idle reaper unloads at runtime). Register
|
||||
// its Close in the cleanup chain — the "caller owns Close" contract
|
||||
// SetLLMService documents.
|
||||
if llmSvc := srv.LLMService(); llmSvc != nil {
|
||||
s.cleanup = append(s.cleanup, func() { _ = llmSvc.Close() })
|
||||
}
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// goTypesEnrichEnabled reports whether the in-process go/packages "go-types"
|
||||
// enrichment provider should be registered for Go. Default ON: it is both
|
||||
// faster and more complete than driving gopls over LSP (see the registration
|
||||
// site), and registration is separately gated on the toolchain being present.
|
||||
// Precedence: GORTEX_GO_TYPES=1/0 wins, then an explicit semantic.go_types
|
||||
// config value, then the on-by-default.
|
||||
func goTypesEnrichEnabled(sem config.SemanticConfig) bool {
|
||||
if v := os.Getenv("GORTEX_GO_TYPES"); v != "" {
|
||||
return v == "1" || strings.EqualFold(v, "true")
|
||||
}
|
||||
if sem.GoTypes != nil {
|
||||
return *sem.GoTypes
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// goTypesIncludeTests reports whether the go-types provider should load
|
||||
// _test.go files (and each package's external test variant). Loading them
|
||||
// roughly doubles the go/packages work per module, so it is OFF by default;
|
||||
// GORTEX_GO_TYPES_TESTS=1 opts in to stamp test-file symbols too (LSP hover
|
||||
// covers them, so the opt-in closes that coverage gap when it matters).
|
||||
func goTypesIncludeTests() bool {
|
||||
return os.Getenv("GORTEX_GO_TYPES_TESTS") == "1" ||
|
||||
strings.EqualFold(os.Getenv("GORTEX_GO_TYPES_TESTS"), "true")
|
||||
}
|
||||
|
||||
// eagerLSPEnabled reports whether the subprocess LSP servers run during the
|
||||
// synchronous enrichment pass. Default OFF — LSP is the slowest part of a cold
|
||||
// index and its net-new value over the in-process tiers (go-types for Go, the
|
||||
// tree-sitter floor for every language) is narrow, so it is kept off the
|
||||
// cold/warm-start path and the router lazy-spawns a server on demand instead.
|
||||
// GORTEX_LSP_EAGER=1 (or semantic.eager_lsp in config) restores the eager
|
||||
// behaviour.
|
||||
func eagerLSPEnabled(sem config.SemanticConfig) bool {
|
||||
if v := os.Getenv("GORTEX_LSP_EAGER"); v != "" {
|
||||
return v == "1" || strings.EqualFold(v, "true")
|
||||
}
|
||||
return sem.EagerLSP
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package serverstack
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/zzet/gortex/internal/config"
|
||||
)
|
||||
|
||||
// TestNewSharedServer_OneshotMemory asserts the shared constructor builds
|
||||
// a working stack over a tmp repo: the graph indexes, and the engine /
|
||||
// MCP server / overlay manager are wired. This is the single-construction-
|
||||
// path validation.
|
||||
func TestNewSharedServer_OneshotMemory(t *testing.T) {
|
||||
repo := t.TempDir()
|
||||
src := "package toy\n\nfunc Add(a, b int) int { return a + b }\nfunc Mul(a, b int) int { return a * b }\n"
|
||||
if err := os.WriteFile(filepath.Join(repo, "toy.go"), []byte(src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ss, err := NewSharedServer(SharedServerConfig{
|
||||
Lifecycle: LifecycleOneshot,
|
||||
Index: repo,
|
||||
Config: config.Default(),
|
||||
Logger: zap.NewNop(),
|
||||
Version: "test",
|
||||
SideStores: SideStores{NotesDir: t.TempDir(), NotesRepo: "test"},
|
||||
// Pin the savings ledger + legacy-import probe to temp paths:
|
||||
// with both empty the constructor opens the REAL machine-global
|
||||
// sidecar and imports (renaming!) the developer's real flat-file
|
||||
// ledger — a unit test must never mutate ~/.gortex.
|
||||
SavingsPath: filepath.Join(t.TempDir(), "sidecar.sqlite"),
|
||||
SavingsLegacyJSON: filepath.Join(t.TempDir(), "savings.json"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewSharedServer: %v", err)
|
||||
}
|
||||
defer ss.Close()
|
||||
|
||||
if ss.Graph == nil || ss.Indexer == nil || ss.Engine == nil || ss.MCP == nil {
|
||||
t.Fatalf("incomplete stack: graph=%v idx=%v eng=%v mcp=%v", ss.Graph != nil, ss.Indexer != nil, ss.Engine != nil, ss.MCP != nil)
|
||||
}
|
||||
if ss.Overlays == nil {
|
||||
t.Error("overlay manager should be wired")
|
||||
}
|
||||
|
||||
if _, err := ss.Indexer.Index(repo); err != nil {
|
||||
t.Fatalf("Index: %v", err)
|
||||
}
|
||||
if n := ss.Graph.Stats().TotalNodes; n == 0 {
|
||||
t.Fatal("graph should be non-empty after indexing the tmp repo")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewSharedServer_OneshotDefaultsMemory asserts the oneshot lifecycle
|
||||
// resolves to the memory backend when Backend is empty.
|
||||
func TestNewSharedServer_OneshotDefaultsMemory(t *testing.T) {
|
||||
if got := LifecycleOneshot.defaultBackend(); got != "memory" {
|
||||
t.Errorf("oneshot default backend = %q, want memory", got)
|
||||
}
|
||||
if got := LifecycleDaemon.defaultBackend(); got != "sqlite" {
|
||||
t.Errorf("daemon default backend = %q, want sqlite", got)
|
||||
}
|
||||
if got := LifecycleHTTP.defaultBackend(); got != "sqlite" {
|
||||
t.Errorf("http default backend = %q, want sqlite", got)
|
||||
}
|
||||
if LifecycleOneshot.Writable() {
|
||||
t.Error("oneshot must not be writable (no store lock)")
|
||||
}
|
||||
if !LifecycleDaemon.Writable() || !LifecycleHTTP.Writable() {
|
||||
t.Error("daemon/http lifecycles own a durable store")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user