chore: import upstream snapshot with attribution

internal/embedding/api.go

@@ -0,0 +1,396 @@
+package embedding
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+)
+
+// maxRetryAfterWait caps how long the API provider will sleep on an
+// HTTP 429 Retry-After hint. A hostile or mis-set header should not be
+// able to stall indexing indefinitely; past this the request just
+// fails and the caller aborts to text-only search.
+const maxRetryAfterWait = 60 * time.Second
+
+// maxEmbedInputBytes caps each embedding input. OpenAI's embedding models
+// reject inputs over 8192 tokens with a 400 that aborts the WHOLE batch
+// (and the vector index). A BPE tokenizer never emits more tokens than
+// input characters, and for single-byte (ASCII) source — the overwhelming
+// majority of code — characters equal bytes, so capping the head at 8000
+// bytes guarantees ≤8000 tokens, safely under the 8192 limit, regardless
+// of how token-dense the snippet is. The truncated head still carries the
+// symbol's signature and leading body — enough signal for nearest-neighbour
+// search. Head-truncation beats dropping the whole index over a few giant
+// generated symbols.
+const maxEmbedInputBytes = 8000
+
+// truncateEmbedInputs head-truncates any input over the byte cap, on a
+// UTF-8 rune boundary so the JSON payload stays valid. Returns the same
+// slice when nothing needed trimming (the common case).
+func truncateEmbedInputs(texts []string) []string {
+	var out []string
+	for i, t := range texts {
+		if len(t) <= maxEmbedInputBytes {
+			continue
+		}
+		if out == nil {
+			out = make([]string, len(texts))
+			copy(out, texts)
+		}
+		b := []byte(t[:maxEmbedInputBytes])
+		for len(b) > 0 && b[len(b)-1]&0xC0 == 0x80 { // back off mid-rune
+			b = b[:len(b)-1]
+		}
+		out[i] = string(b)
+	}
+	if out == nil {
+		return texts
+	}
+	return out
+}
+
+// APIProvider calls an external embedding API (Ollama or OpenAI-compatible).
+type APIProvider struct {
+	url    string
+	model  string
+	apiKey string
+	client *http.Client
+	dims   int
+	format apiFormat
+
+	// tokensUsed accumulates the `usage.total_tokens` reported by the
+	// embedding backend across every request, so the indexer can log the
+	// actual token spend of a paid embedding pass (otherwise invisible).
+	// Touched from several goroutines under the concurrent embedding pool,
+	// hence atomic.
+	tokensUsed int64
+}
+
+// TokensUsed reports the total embedding tokens this provider has been
+// billed for so far, summed from each response's usage.total_tokens.
+// Returns 0 for backends that don't report usage (e.g. Ollama).
+func (p *APIProvider) TokensUsed() int64 { return atomic.LoadInt64(&p.tokensUsed) }
+
+type apiFormat int
+
+const (
+	formatOllama apiFormat = iota
+	formatOpenAI
+)
+
+// NewAPIProvider creates a provider that calls an external embedding API.
+// Auto-detects Ollama vs OpenAI format from the URL.
+func NewAPIProvider(url, model string) *APIProvider {
+	format := formatOpenAI
+	if strings.Contains(url, "11434") || strings.Contains(url, "/api/") {
+		format = formatOllama
+	}
+
+	if model == "" {
+		if format == formatOllama {
+			model = "nomic-embed-text"
+		} else {
+			model = "text-embedding-3-small"
+		}
+	}
+
+	// API key for authenticated embedding backends (OpenAI, Azure, and
+	// OpenAI-compatible gateways). Ollama on localhost is keyless, so the
+	// key stays optional and an unset value just omits the header. Prefer
+	// an explicit GORTEX_EMBEDDINGS_API_KEY; fall back to OPENAI_API_KEY
+	// only when the endpoint is api.openai.com, so a stray OPENAI_API_KEY
+	// can never leak to an arbitrary third-party URL.
+	apiKey := os.Getenv("GORTEX_EMBEDDINGS_API_KEY")
+	if apiKey == "" && strings.Contains(url, "openai.com") {
+		apiKey = os.Getenv("OPENAI_API_KEY")
+	}
+
+	return &APIProvider{
+		url:    strings.TrimRight(url, "/"),
+		model:  model,
+		apiKey: apiKey,
+		client: &http.Client{Timeout: 30 * time.Second},
+		format: format,
+	}
+}
+
+func (p *APIProvider) Embed(ctx context.Context, text string) ([]float32, error) {
+	vecs, err := p.EmbedBatch(ctx, []string{text})
+	if err != nil {
+		return nil, err
+	}
+	if len(vecs) == 0 {
+		return nil, fmt.Errorf("embedding API returned no results")
+	}
+	return vecs[0], nil
+}
+
+func (p *APIProvider) EmbedBatch(ctx context.Context, texts []string) ([][]float32, error) {
+	var (
+		vecs [][]float32
+		err  error
+	)
+	if p.format == formatOllama {
+		vecs, err = p.embedOllama(ctx, texts)
+	} else {
+		vecs, err = p.embedOpenAI(ctx, texts)
+	}
+	if err != nil {
+		return nil, err
+	}
+	// Width is learned lazily from the response (dims 0), so validate the count,
+	// reject empty vectors, and check the batch is internally consistent without
+	// asserting an absolute width.
+	if err := validateBatch("api", texts, vecs, 0); err != nil {
+		return nil, err
+	}
+	return vecs, nil
+}
+
+func (p *APIProvider) Dimensions() int { return p.dims }
+func (p *APIProvider) Close() error    { return nil }
+
+// ProbeDimensions makes one tiny embedding call to discover and cache the
+// provider's vector width, so Dimensions() reports the true value *before*
+// the first indexing pass. An APIProvider learns its width only from the
+// first real embed (embedOpenAI / embedOllama set p.dims from the returned
+// vector); until then Dimensions() returns 0, which has two concrete
+// consequences at daemon startup: the "embeddings enabled" log mislabels
+// the width as dim:0, and the snapshot-vector reload gate
+// (daemon_state.go: vec.Dims == EmbedderDims) rejects a correctly-sized
+// persisted index, forcing a needless full re-embed on every restart.
+//
+// Idempotent: a no-op once the width is known. Best-effort: on any
+// transport/auth error it returns the error and leaves dims at 0 — the
+// caller logs a warning, the lazy path still sets the width from the first
+// real vector, and indexing degrades to BM25 if embeddings are truly
+// unreachable. The probe also doubles as an early connectivity/credential
+// check, surfacing a bad key or URL at startup instead of mid-index.
+func (p *APIProvider) ProbeDimensions(ctx context.Context) (int, error) {
+	if d := p.Dimensions(); d > 0 {
+		return d, nil
+	}
+	pctx, cancel := context.WithTimeout(ctx, 20*time.Second)
+	defer cancel()
+	vec, err := p.Embed(pctx, "gortex embedding dimension probe")
+	if err != nil {
+		return 0, err
+	}
+	// Embed -> EmbedBatch -> embed{OpenAI,Ollama} already cached p.dims from
+	// the response; fall back to the returned vector's length defensively.
+	if p.dims == 0 && len(vec) > 0 {
+		p.dims = len(vec)
+	}
+	return p.dims, nil
+}
+
+// Concurrent reports that this provider is safe — and worth — calling
+// from several goroutines at once. An external HTTP embedding endpoint
+// gains from overlapped round-trips; the indexer's embedding pool uses
+// this to decide whether to parallelise.
+func (p *APIProvider) Concurrent() bool { return true }
+
+// doRequest issues req via the provider's HTTP client and returns the
+// response. On an HTTP 429 it honours a Retry-After header (delta-
+// seconds form) and retries once after sleeping — capped at
+// maxRetryAfterWait so a bad header cannot stall indexing. The caller
+// owns closing the returned body.
+func (p *APIProvider) doRequest(ctx context.Context, req *http.Request, body []byte) (*http.Response, error) {
+	resp, err := p.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusTooManyRequests {
+		return resp, nil
+	}
+	// Rate limited — read the hint, drain and close this response,
+	// then retry exactly once.
+	wait := parseRetryAfter(resp.Header.Get("Retry-After"))
+	_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1024))
+	_ = resp.Body.Close()
+	if wait <= 0 {
+		// No usable hint — let the caller surface the 429 by re-issuing
+		// once without a sleep would just hammer the API, so fall back
+		// to a short fixed backoff.
+		wait = time.Second
+	}
+	if wait > maxRetryAfterWait {
+		wait = maxRetryAfterWait
+	}
+	select {
+	case <-time.After(wait):
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+	// Rebuild the request — a *http.Request body is single-use.
+	retry, err := http.NewRequestWithContext(ctx, req.Method, req.URL.String(), bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	retry.Header = req.Header.Clone()
+	return p.client.Do(retry)
+}
+
+// parseRetryAfter parses the delta-seconds form of an HTTP Retry-After
+// header ("Retry-After: 12"). The HTTP-date form is not handled —
+// embedding APIs use delta-seconds in practice — and returns 0, which
+// the caller treats as "no usable hint".
+func parseRetryAfter(v string) time.Duration {
+	v = strings.TrimSpace(v)
+	if v == "" {
+		return 0
+	}
+	if secs, err := strconv.Atoi(v); err == nil && secs >= 0 {
+		return time.Duration(secs) * time.Second
+	}
+	return 0
+}
+
+// --- Ollama API ---
+
+type ollamaRequest struct {
+	Model string `json:"model"`
+	Input any    `json:"input"` // string or []string
+}
+
+type ollamaResponse struct {
+	Embeddings [][]float32 `json:"embeddings"`
+}
+
+func (p *APIProvider) embedOllama(ctx context.Context, texts []string) ([][]float32, error) {
+	reqBody := ollamaRequest{
+		Model: p.model,
+		Input: truncateEmbedInputs(texts),
+	}
+
+	body, err := json.Marshal(reqBody)
+	if err != nil {
+		return nil, fmt.Errorf("marshal request: %w", err)
+	}
+
+	url := p.url + "/api/embed"
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if p.apiKey != "" {
+		req.Header.Set("Authorization", "Bearer "+p.apiKey)
+	}
+
+	resp, err := p.doRequest(ctx, req, body)
+	if err != nil {
+		return nil, fmt.Errorf("API call: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
+		return nil, fmt.Errorf("API error %d: %s", resp.StatusCode, string(respBody))
+	}
+
+	var result ollamaResponse
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, fmt.Errorf("decode response: %w", err)
+	}
+
+	if len(result.Embeddings) > 0 && p.dims == 0 {
+		p.dims = len(result.Embeddings[0])
+	}
+
+	return result.Embeddings, nil
+}
+
+// --- OpenAI API ---
+
+type openAIRequest struct {
+	Model string   `json:"model"`
+	Input []string `json:"input"`
+}
+
+type openAIResponse struct {
+	Data  []openAIEmbedding `json:"data"`
+	Usage openAIUsage       `json:"usage"`
+}
+
+// openAIUsage carries the token accounting OpenAI returns alongside every
+// embeddings response. total_tokens is what the request is billed on.
+type openAIUsage struct {
+	PromptTokens int `json:"prompt_tokens"`
+	TotalTokens  int `json:"total_tokens"`
+}
+
+type openAIEmbedding struct {
+	Embedding []float32 `json:"embedding"`
+	Index     int       `json:"index"`
+}
+
+func (p *APIProvider) embedOpenAI(ctx context.Context, texts []string) ([][]float32, error) {
+	reqBody := openAIRequest{
+		Model: p.model,
+		Input: truncateEmbedInputs(texts),
+	}
+
+	body, err := json.Marshal(reqBody)
+	if err != nil {
+		return nil, fmt.Errorf("marshal request: %w", err)
+	}
+
+	// OpenAI-compatible bases are conventionally given WITH the version
+	// segment (OpenAI "https://api.openai.com/v1", OpenRouter
+	// "https://openrouter.ai/api/v1"). Append "/v1" only when it is absent,
+	// so a "…/v1" base does not become "…/v1/v1/embeddings" (a 404 that
+	// silently degrades the whole vector index to BM25).
+	endpoint := "/v1/embeddings"
+	if strings.HasSuffix(p.url, "/v1") {
+		endpoint = "/embeddings"
+	}
+	url := p.url + endpoint
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if p.apiKey != "" {
+		req.Header.Set("Authorization", "Bearer "+p.apiKey)
+	}
+
+	resp, err := p.doRequest(ctx, req, body)
+	if err != nil {
+		return nil, fmt.Errorf("API call: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
+		return nil, fmt.Errorf("API error %d: %s", resp.StatusCode, string(respBody))
+	}
+
+	var result openAIResponse
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, fmt.Errorf("decode response: %w", err)
+	}
+
+	if result.Usage.TotalTokens > 0 {
+		atomic.AddInt64(&p.tokensUsed, int64(result.Usage.TotalTokens))
+	}
+
+	vecs := make([][]float32, len(result.Data))
+	for _, d := range result.Data {
+		vecs[d.Index] = d.Embedding
+	}
+
+	if len(vecs) > 0 && p.dims == 0 && len(vecs[0]) > 0 {
+		p.dims = len(vecs[0])
+	}
+
+	return vecs, nil
+}

internal/embedding/api_test.go

@@ -0,0 +1,286 @@
+package embedding
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestAPIProvider_Concurrent asserts the API provider declares itself
+// safe to call concurrently — the signal the embedding pool gates on.
+func TestAPIProvider_Concurrent(t *testing.T) {
+	p := NewAPIProvider("http://localhost:11434", "")
+	assert.True(t, p.Concurrent(), "the API provider must opt into concurrent embedding")
+}
+
+// TestParseRetryAfter covers the delta-seconds Retry-After parser.
+func TestParseRetryAfter(t *testing.T) {
+	assert.Equal(t, 12*time.Second, parseRetryAfter("12"))
+	assert.Equal(t, time.Duration(0), parseRetryAfter(""))
+	assert.Equal(t, time.Duration(0), parseRetryAfter("  "))
+	assert.Equal(t, time.Duration(0), parseRetryAfter("Wed, 21 Oct 2026 07:28:00 GMT"),
+		"the HTTP-date form is not parsed — returns 0 so the caller uses a fixed backoff")
+	assert.Equal(t, time.Duration(0), parseRetryAfter("-5"), "a negative hint is rejected")
+}
+
+// TestAPIProvider_HonorsRetryAfterOn429 asserts the provider retries
+// once after an HTTP 429, honouring the Retry-After header, and
+// succeeds when the retry returns 200.
+func TestAPIProvider_HonorsRetryAfterOn429(t *testing.T) {
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := atomic.AddInt32(&calls, 1)
+		if n == 1 {
+			// First call: rate-limited with a 1-second hint.
+			w.Header().Set("Retry-After", "1")
+			w.WriteHeader(http.StatusTooManyRequests)
+			return
+		}
+		// Retry: succeed with an OpenAI-shaped embedding response.
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(openAIResponse{
+			Data: []openAIEmbedding{{Embedding: []float32{1, 2, 3}, Index: 0}},
+		})
+	}))
+	defer srv.Close()
+
+	// srv.URL has no Ollama markers, so the provider uses OpenAI format.
+	p := NewAPIProvider(srv.URL, "text-embedding-3-small")
+
+	start := time.Now()
+	vecs, err := p.EmbedBatch(context.Background(), []string{"hello"})
+	require.NoError(t, err, "the embed must succeed after the post-429 retry")
+	require.Len(t, vecs, 1)
+	assert.Equal(t, []float32{1, 2, 3}, vecs[0])
+	assert.Equal(t, int32(2), atomic.LoadInt32(&calls), "exactly one retry after the 429")
+	assert.GreaterOrEqual(t, time.Since(start), time.Second,
+		"the provider must wait out the Retry-After hint before retrying")
+}
+
+// TestAPIProvider_429WithoutHintStillRetries asserts that a 429 with no
+// Retry-After header still triggers exactly one retry (on a short fixed
+// backoff) rather than failing immediately.
+func TestAPIProvider_429WithoutHintStillRetries(t *testing.T) {
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if atomic.AddInt32(&calls, 1) == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(openAIResponse{
+			Data: []openAIEmbedding{{Embedding: []float32{4, 5, 6}, Index: 0}},
+		})
+	}))
+	defer srv.Close()
+
+	p := NewAPIProvider(srv.URL, "")
+	vecs, err := p.EmbedBatch(context.Background(), []string{"x"})
+	require.NoError(t, err)
+	require.Len(t, vecs, 1)
+	assert.Equal(t, int32(2), atomic.LoadInt32(&calls))
+}
+
+// TestAPIProvider_PersistentRateLimitFails asserts a server that keeps
+// returning 429 eventually surfaces an error — the retry is bounded to
+// one attempt, it is not an infinite loop.
+func TestAPIProvider_PersistentRateLimitFails(t *testing.T) {
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusTooManyRequests)
+	}))
+	defer srv.Close()
+
+	p := NewAPIProvider(srv.URL, "")
+	_, err := p.EmbedBatch(context.Background(), []string{"x"})
+	require.Error(t, err, "a persistent 429 must eventually fail")
+	assert.LessOrEqual(t, atomic.LoadInt32(&calls), int32(2),
+		"the 429 retry is bounded to a single re-attempt")
+}
+
+// TestAPIProvider_SendsAuthorizationHeader asserts that an embeddings API
+// key (GORTEX_EMBEDDINGS_API_KEY) is forwarded as an Authorization: Bearer
+// header — the fix that lets gortex use authenticated backends like OpenAI.
+func TestAPIProvider_SendsAuthorizationHeader(t *testing.T) {
+	t.Setenv("GORTEX_EMBEDDINGS_API_KEY", "test-secret")
+
+	var gotAuth string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotAuth = r.Header.Get("Authorization")
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"data":[{"embedding":[0.1,0.2],"index":0}]}`))
+	}))
+	defer srv.Close()
+
+	// A non-Ollama URL selects the OpenAI format (the /v1/embeddings path).
+	p := NewAPIProvider(srv.URL, "text-embedding-3-small")
+	_, err := p.EmbedBatch(context.Background(), []string{"hello"})
+	require.NoError(t, err)
+	assert.Equal(t, "Bearer test-secret", gotAuth)
+}
+
+// TestAPIProvider_NoKeyOmitsAuthorizationHeader asserts that with no key
+// configured, no Authorization header is sent (keyless Ollama stays keyless
+// and a stray OPENAI_API_KEY does not leak to a non-OpenAI endpoint).
+func TestAPIProvider_NoKeyOmitsAuthorizationHeader(t *testing.T) {
+	t.Setenv("GORTEX_EMBEDDINGS_API_KEY", "")
+	t.Setenv("OPENAI_API_KEY", "")
+
+	var hadAuth bool
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, hadAuth = r.Header["Authorization"]
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"data":[{"embedding":[0.1],"index":0}]}`))
+	}))
+	defer srv.Close()
+
+	p := NewAPIProvider(srv.URL, "text-embedding-3-small")
+	_, err := p.EmbedBatch(context.Background(), []string{"hi"})
+	require.NoError(t, err)
+	assert.False(t, hadAuth, "no Authorization header without a configured key")
+}
+
+// TestAPIProvider_AccumulatesTokenUsage asserts the provider reads the
+// OpenAI `usage.total_tokens` field off each embedding response and
+// accumulates it across calls — the signal the indexer logs so the paid
+// embedding pass reports its actual token spend (it otherwise vanishes).
+func TestAPIProvider_AccumulatesTokenUsage(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"data":[{"embedding":[0.1,0.2],"index":0}],"usage":{"prompt_tokens":7,"total_tokens":7}}`))
+	}))
+	defer srv.Close()
+
+	p := NewAPIProvider(srv.URL, "text-embedding-3-small")
+	assert.Equal(t, int64(0), p.TokensUsed(), "no tokens before any call")
+
+	_, err := p.EmbedBatch(context.Background(), []string{"hello"})
+	require.NoError(t, err)
+	_, err = p.EmbedBatch(context.Background(), []string{"world"})
+	require.NoError(t, err)
+
+	assert.Equal(t, int64(14), p.TokensUsed(), "usage accumulates across batches")
+}
+
+// TestAPIProvider_OpenAIBaseURLVariants asserts the OpenAI request path is
+// "/v1/embeddings" whether the base URL is given with or without the "/v1"
+// version segment. OpenAI-compatible bases conventionally include "/v1"
+// (OpenAI, OpenRouter); without this normalization a "…/v1" base produced
+// "…/v1/v1/embeddings" → 404 → silent fallback to BM25.
+func TestAPIProvider_OpenAIBaseURLVariants(t *testing.T) {
+	for _, suffix := range []string{"", "/v1"} {
+		suffix := suffix
+		t.Run("base"+suffix, func(t *testing.T) {
+			var gotPath string
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				gotPath = r.URL.Path
+				w.Header().Set("Content-Type", "application/json")
+				_, _ = w.Write([]byte(`{"data":[{"embedding":[0.1,0.2],"index":0}]}`))
+			}))
+			defer srv.Close()
+
+			p := NewAPIProvider(srv.URL+suffix, "text-embedding-3-small")
+			_, err := p.EmbedBatch(context.Background(), []string{"hi"})
+			require.NoError(t, err)
+			assert.Equal(t, "/v1/embeddings", gotPath,
+				"both base forms must resolve to a single /v1/embeddings path")
+		})
+	}
+}
+
+// TestAPIProvider_ProbeDimensions asserts the probe discovers the vector
+// width with exactly one embedding call, caches it (so Dimensions() then
+// reports the true width up front), and is idempotent — a second probe
+// issues no further HTTP. This is the fix for the daemon logging dim:0 and
+// the snapshot-vector reload gate rejecting a correctly-sized cached index.
+func TestAPIProvider_ProbeDimensions(t *testing.T) {
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.Header().Set("Content-Type", "application/json")
+		// A 4-dimensional vector stands in for OpenAI's 1536-d response.
+		_, _ = w.Write([]byte(`{"data":[{"embedding":[0.1,0.2,0.3,0.4],"index":0}],"usage":{"total_tokens":3}}`))
+	}))
+	defer srv.Close()
+
+	p := NewAPIProvider(srv.URL, "text-embedding-3-small")
+	assert.Equal(t, 0, p.Dimensions(), "width unknown before the first call")
+
+	dim, err := p.ProbeDimensions(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 4, dim, "probe reports the response vector width")
+	assert.Equal(t, 4, p.Dimensions(), "width is cached after the probe")
+	assert.Equal(t, int32(1), atomic.LoadInt32(&calls), "the probe is a single call")
+
+	dim2, err := p.ProbeDimensions(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 4, dim2)
+	assert.Equal(t, int32(1), atomic.LoadInt32(&calls), "a second probe issues no further HTTP")
+}
+
+// TestAPIProvider_ProbeDimensionsError asserts that a probe against an
+// unreachable / erroring backend surfaces the error and leaves the width at
+// 0 (best-effort) — the caller only warns and indexing falls back to BM25.
+func TestAPIProvider_ProbeDimensionsError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusUnauthorized)
+		_, _ = w.Write([]byte(`{"error":"bad key"}`))
+	}))
+	defer srv.Close()
+
+	p := NewAPIProvider(srv.URL, "text-embedding-3-small")
+	dim, err := p.ProbeDimensions(context.Background())
+	require.Error(t, err, "an auth failure must surface as an error")
+	assert.Equal(t, 0, dim)
+	assert.Equal(t, 0, p.Dimensions(), "a failed probe leaves the width unset")
+}
+
+// TestAPIProvider_ProbeDimensionsLive hits the REAL OpenAI embeddings API to
+// prove the fork's embedder is genuinely wired end-to-end (not a stub): the
+// probe returns text-embedding-3-small's true 1536-d width, a batch embed
+// returns 1536-d vectors, and token usage is accounted. Skipped unless a key
+// is present so CI without credentials stays green.
+//
+//	OPENAI_API_KEY=sk-... go test ./internal/embedding -run ProbeDimensionsLive -v
+func TestAPIProvider_ProbeDimensionsLive(t *testing.T) {
+	if os.Getenv("GORTEX_EMBEDDINGS_API_KEY") == "" && os.Getenv("OPENAI_API_KEY") == "" {
+		t.Skip("no embeddings API key (set OPENAI_API_KEY) — skipping live OpenAI probe")
+	}
+
+	p := NewAPIProvider("https://api.openai.com/v1", "text-embedding-3-small")
+	dim, err := p.ProbeDimensions(context.Background())
+	require.NoError(t, err, "live probe against OpenAI must succeed")
+	assert.Equal(t, 1536, dim, "text-embedding-3-small is 1536-dimensional")
+	assert.Equal(t, 1536, p.Dimensions())
+
+	vecs, err := p.EmbedBatch(context.Background(), []string{"rule engine evaluates facts", "knowledge base"})
+	require.NoError(t, err)
+	require.Len(t, vecs, 2)
+	assert.Len(t, vecs[0], 1536, "each returned vector is 1536-d")
+	assert.Len(t, vecs[1], 1536)
+	assert.Greater(t, p.TokensUsed(), int64(0), "the paid embedding pass accounts token usage")
+}
+
+// TestTruncateEmbedInputs asserts oversized inputs are head-truncated to the
+// byte cap (so OpenAI's 8192-token limit can't abort the whole vector index)
+// while normal inputs pass through untouched.
+func TestTruncateEmbedInputs(t *testing.T) {
+	short := "small symbol"
+	long := string(make([]byte, maxEmbedInputBytes+500))
+
+	out := truncateEmbedInputs([]string{short, long})
+	assert.Equal(t, short, out[0], "short input untouched")
+	assert.LessOrEqual(t, len(out[1]), maxEmbedInputBytes, "long input capped")
+
+	in := []string{"a", "b"}
+	assert.Equal(t, in, truncateEmbedInputs(in), "no oversize → same slice values")
+}

internal/embedding/chunk.go

@@ -0,0 +1,568 @@
+package embedding
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	sitter "github.com/zzet/gortex/internal/parser/tsitter"
+	clang "github.com/zzet/gortex/internal/parser/tsitter/c"
+	cpplang "github.com/zzet/gortex/internal/parser/tsitter/cpp"
+	golang "github.com/zzet/gortex/internal/parser/tsitter/golang"
+	javalang "github.com/zzet/gortex/internal/parser/tsitter/java"
+	jslang "github.com/zzet/gortex/internal/parser/tsitter/javascript"
+	kotlinlang "github.com/zzet/gortex/internal/parser/tsitter/kotlin"
+	phplang "github.com/zzet/gortex/internal/parser/tsitter/php"
+	pylang "github.com/zzet/gortex/internal/parser/tsitter/python"
+	rubylang "github.com/zzet/gortex/internal/parser/tsitter/ruby"
+	rustlang "github.com/zzet/gortex/internal/parser/tsitter/rust"
+	swiftlang "github.com/zzet/gortex/internal/parser/tsitter/swift"
+	tsxlang "github.com/zzet/gortex/internal/parser/tsitter/tsx"
+	tslang "github.com/zzet/gortex/internal/parser/tsitter/typescript"
+)
+
+// Chunk is one AST window cut out of a symbol's source span. A symbol
+// short enough to embed whole produces exactly one Chunk; a large
+// function or type produces several, each covering a contiguous run of
+// top-level statements / field declarations.
+type Chunk struct {
+	// Text is the chunk's source text — the substring of the symbol's
+	// span the window covers. It is what gets embedded.
+	Text string
+	// ParentID is the graph node ID of the symbol the chunk belongs to.
+	// Every chunk of a symbol carries the same ParentID; the de-chunk
+	// step at query time maps a chunk hit back through it.
+	ParentID string
+	// WindowIndex is the 0-based position of this window within the
+	// symbol. A single-chunk symbol has WindowIndex 0.
+	WindowIndex int
+}
+
+// ChunkOptions tunes the AST-window splitter.
+type ChunkOptions struct {
+	// ThresholdLines is the line count above which a symbol is split
+	// into windows. At or below it the symbol is embedded whole.
+	ThresholdLines int
+	// WindowLines caps the line span of each emitted window. A single
+	// top-level statement larger than this still forms its own window
+	// (the splitter never cuts inside a statement).
+	WindowLines int
+}
+
+const (
+	// DefaultChunkThresholdLines is the built-in split threshold used
+	// when ChunkOptions.ThresholdLines is zero.
+	DefaultChunkThresholdLines = 60
+	// DefaultChunkWindowLines is the built-in window cap used when
+	// ChunkOptions.WindowLines is zero.
+	DefaultChunkWindowLines = 40
+	// chunkParseTimeout bounds the tree-sitter parse of one symbol's
+	// span. Generous — a symbol body is small, but a pathological
+	// grammar should still not stall the index pass.
+	chunkParseTimeout = 3 * time.Second
+)
+
+// normalized fills in zero-valued options with the package defaults.
+func (o ChunkOptions) normalized() ChunkOptions {
+	if o.ThresholdLines <= 0 {
+		o.ThresholdLines = DefaultChunkThresholdLines
+	}
+	if o.WindowLines <= 0 {
+		o.WindowLines = DefaultChunkWindowLines
+	}
+	// A window can never be smaller than a single line, and a window
+	// larger than the threshold would never split anything.
+	if o.WindowLines < 1 {
+		o.WindowLines = 1
+	}
+	return o
+}
+
+// ChunkSymbol splits a symbol's source span into AST windows. src is
+// the exact source text of the symbol (signature through closing
+// brace). language is the tree-sitter language name. parentID is the
+// graph node ID stamped on every returned chunk.
+//
+// The result is always non-empty. A symbol at or below the line
+// threshold, in a language with no splitter, or whose source fails to
+// parse, yields a single chunk holding the whole span. A large
+// function is split on the top-level statements of its body; a large
+// type on its field declarations. Windows never cut inside a
+// statement, so one oversized statement forms its own window.
+func ChunkSymbol(src []byte, language, parentID string, opts ChunkOptions) []Chunk {
+	opts = opts.normalized()
+	whole := []Chunk{{Text: string(src), ParentID: parentID, WindowIndex: 0}}
+
+	if len(src) == 0 {
+		return whole
+	}
+	if countLines(src) <= opts.ThresholdLines {
+		return whole
+	}
+	spec := chunkSpecFor(language)
+	if spec == nil {
+		return whole
+	}
+	grammar := spec.grammar()
+	if grammar == nil {
+		return whole
+	}
+
+	parser := sitter.NewParser()
+	defer parser.Close()
+	parser.SetLanguage(grammar)
+	ctx, cancel := context.WithTimeout(context.Background(), chunkParseTimeout)
+	defer cancel()
+	tree, err := parser.ParseCtx(ctx, nil, src)
+	if err != nil || tree == nil {
+		return whole
+	}
+	defer tree.Close()
+	root := tree.RootNode()
+	if root == nil {
+		return whole
+	}
+
+	// Locate the splittable container — a function/method body block or
+	// a type's field-declaration list — and collect the byte ranges of
+	// its top-level children.
+	container := spec.findContainer(root)
+	if container == nil {
+		return whole
+	}
+	container = unwrapStatementList(container)
+	pieces := topLevelChildRanges(container)
+	if len(pieces) < 2 {
+		// Nothing to split into more than one window.
+		return whole
+	}
+
+	windows := packWindows(src, pieces, opts.WindowLines)
+	if len(windows) < 2 {
+		return whole
+	}
+	chunks := make([]Chunk, len(windows))
+	for i, w := range windows {
+		chunks[i] = Chunk{Text: w, ParentID: parentID, WindowIndex: i}
+	}
+	return chunks
+}
+
+// byteRange is a half-open [start,end) byte span within the symbol src.
+type byteRange struct {
+	start, end uint32
+}
+
+// unwrapStatementList descends through wrapper nodes that hold the
+// real split points one level deeper. Tree-sitter-go wraps a function
+// body's statements in a `statement_list` inside the `block`; the
+// `block` itself then has only that one named child. Unwrapping it
+// (and any chain of such single-child list wrappers) exposes the
+// statements as the container's direct children. A wrapper with more
+// than one named child, or a non-list child, is left as the container.
+func unwrapStatementList(container *sitter.Node) *sitter.Node {
+	for i := 0; i < 4; i++ { // bounded — real grammars nest at most once
+		if container == nil || container.NamedChildCount() != 1 {
+			return container
+		}
+		only := container.NamedChild(0)
+		if only == nil {
+			return container
+		}
+		t := only.Type()
+		if t == "statement_list" || strings.HasSuffix(t, "_declaration_list") {
+			container = only
+			continue
+		}
+		return container
+	}
+	return container
+}
+
+// topLevelChildRanges returns the byte ranges of the named children of
+// a container node (statements of a block, field declarations of a
+// field list). Anonymous tokens (braces, commas) are skipped.
+func topLevelChildRanges(container *sitter.Node) []byteRange {
+	n := int(container.NamedChildCount())
+	ranges := make([]byteRange, 0, n)
+	for i := 0; i < n; i++ {
+		c := container.NamedChild(i)
+		if c == nil {
+			continue
+		}
+		ranges = append(ranges, byteRange{start: c.StartByte(), end: c.EndByte()})
+	}
+	return ranges
+}
+
+// packWindows groups consecutive child ranges into windows of at most
+// windowLines lines each, and returns the source text of every window.
+// The first window also captures the symbol's signature (everything
+// before the first child); the last captures the trailing bytes
+// (closing brace) so the rejoined windows still cover the whole span.
+// A single child larger than windowLines forms its own window.
+func packWindows(src []byte, pieces []byteRange, windowLines int) []string {
+	if len(pieces) == 0 {
+		return []string{string(src)}
+	}
+
+	var windows []string
+	groupStart := uint32(0) // first window starts at the symbol's own start (keeps the signature)
+	cur := 0                // index of the first piece in the current group
+	curLines := 0
+
+	flush := func(endByte uint32, upto int) {
+		if upto <= cur {
+			return
+		}
+		windows = append(windows, string(src[groupStart:endByte]))
+		groupStart = endByte
+		cur = upto
+		curLines = 0
+	}
+
+	for i, p := range pieces {
+		pieceLines := countLines(src[p.start:p.end])
+		// If adding this piece would overflow the window and the
+		// window already holds something, close the window before it.
+		if curLines > 0 && curLines+pieceLines > windowLines {
+			flush(pieces[i-1].end, i)
+		}
+		curLines += pieceLines
+	}
+	// Final window runs to the end of the symbol span so the trailing
+	// closing brace is never dropped.
+	if cur < len(pieces) {
+		windows = append(windows, string(src[groupStart:]))
+	}
+	if len(windows) == 0 {
+		return []string{string(src)}
+	}
+	return windows
+}
+
+// countLines returns the number of source lines a byte slice spans (a
+// non-empty slice with no newline is one line).
+func countLines(b []byte) int {
+	if len(b) == 0 {
+		return 0
+	}
+	return strings.Count(string(b), "\n") + 1
+}
+
+// chunkSpec describes how to find a splittable container in one
+// tree-sitter grammar.
+type chunkSpec struct {
+	grammar func() *sitter.Language
+	// findContainer locates the node whose named children are the
+	// split points: a function/method body block, or a type's field
+	// list. Returns nil when the parsed span has no such container.
+	findContainer func(root *sitter.Node) *sitter.Node
+}
+
+// chunkSpecFor returns the splitter spec for a language, or nil when
+// the language has no splitter (the symbol is then embedded whole).
+func chunkSpecFor(language string) *chunkSpec {
+	switch strings.ToLower(language) {
+	case "go", "golang":
+		return &chunkSpec{
+			grammar: golang.GetLanguage,
+			findContainer: braceContainer(
+				containerSpec{decl: "function_declaration", body: "block", bodyField: "body"},
+				containerSpec{decl: "method_declaration", body: "block", bodyField: "body"},
+				// A Go struct: split on its field declarations. The
+				// struct_type node is reached through type_declaration →
+				// type_spec; walkNodes descends to it.
+				containerSpec{decl: "struct_type", body: "field_declaration_list"},
+			),
+		}
+	case "typescript", "ts":
+		return &chunkSpec{
+			grammar:       tslang.GetLanguage,
+			findContainer: tsLikeContainer,
+		}
+	case "tsx":
+		return &chunkSpec{
+			grammar:       tsxlang.GetLanguage,
+			findContainer: tsLikeContainer,
+		}
+	case "javascript", "js", "jsx":
+		return &chunkSpec{
+			grammar:       jslang.GetLanguage,
+			findContainer: tsLikeContainer,
+		}
+	case "java":
+		return &chunkSpec{
+			grammar: javalang.GetLanguage,
+			findContainer: braceContainer(
+				containerSpec{decl: "method_declaration", body: "block", bodyField: "body"},
+				containerSpec{decl: "constructor_declaration", body: "constructor_body", bodyField: "body"},
+				containerSpec{decl: "class_declaration", body: "class_body", bodyField: "body"},
+			),
+		}
+	case "c":
+		return &chunkSpec{
+			grammar: clang.GetLanguage,
+			findContainer: braceContainer(
+				containerSpec{decl: "function_definition", body: "compound_statement", bodyField: "body"},
+			),
+		}
+	case "cpp", "c++":
+		return &chunkSpec{
+			grammar: cpplang.GetLanguage,
+			findContainer: braceContainer(
+				containerSpec{decl: "function_definition", body: "compound_statement", bodyField: "body"},
+			),
+		}
+	case "rust":
+		return &chunkSpec{
+			grammar: rustlang.GetLanguage,
+			findContainer: braceContainer(
+				containerSpec{decl: "function_item", body: "block", bodyField: "body"},
+			),
+		}
+	case "python", "py":
+		return &chunkSpec{
+			grammar:       pylang.GetLanguage,
+			findContainer: pythonContainer,
+		}
+	case "ruby", "rb":
+		return &chunkSpec{
+			grammar:       rubylang.GetLanguage,
+			findContainer: rubyContainer,
+		}
+	case "php":
+		return &chunkSpec{
+			grammar: phplang.GetLanguage,
+			findContainer: braceContainer(
+				containerSpec{decl: "method_declaration", body: "compound_statement", bodyField: "body"},
+				containerSpec{decl: "function_definition", body: "compound_statement", bodyField: "body"},
+				containerSpec{decl: "class_declaration", body: "declaration_list", bodyField: "body"},
+			),
+		}
+	case "kotlin", "kt":
+		return &chunkSpec{
+			grammar:       kotlinlang.GetLanguage,
+			findContainer: kotlinSwiftContainer,
+		}
+	case "swift":
+		return &chunkSpec{
+			grammar:       swiftlang.GetLanguage,
+			findContainer: kotlinSwiftContainer,
+		}
+	default:
+		return nil
+	}
+}
+
+// rubyContainer finds the splittable container in a parsed Ruby span: a
+// method / singleton_method body, or a class / module body. The
+// tree-sitter-ruby grammar models the body as a `body_statement` named
+// child (not a field) whose own named children are the statements —
+// the split points.
+func rubyContainer(root *sitter.Node) *sitter.Node {
+	decls := map[string]struct{}{
+		"method":           {},
+		"singleton_method": {},
+		"class":            {},
+		"module":           {},
+	}
+	var found *sitter.Node
+	walkNodes(root, func(n *sitter.Node) bool {
+		if found != nil {
+			return false
+		}
+		if _, ok := decls[n.Type()]; ok {
+			if body := firstNamedChildOfKind(n, map[string]struct{}{"body_statement": {}}); body != nil {
+				found = body
+				return false
+			}
+		}
+		return true
+	})
+	return found
+}
+
+// kotlinSwiftContainer finds the splittable container in a parsed
+// Kotlin or Swift span. Both grammars share the same shape: a
+// `function_declaration` wraps its block body in a `function_body`
+// node whose single `statements` named child holds the real split
+// points, and a `class_declaration` exposes a `class_body` whose named
+// children (functions, properties) are the split points. The function
+// path returns the inner `statements` node so topLevelChildRanges sees
+// the statements directly; the class path returns `class_body`.
+func kotlinSwiftContainer(root *sitter.Node) *sitter.Node {
+	var found *sitter.Node
+	walkNodes(root, func(n *sitter.Node) bool {
+		if found != nil {
+			return false
+		}
+		switch n.Type() {
+		case "function_declaration":
+			if body := firstNamedChildOfKind(n, map[string]struct{}{"function_body": {}}); body != nil {
+				if stmts := firstNamedChildOfKind(body, map[string]struct{}{"statements": {}}); stmts != nil {
+					found = stmts
+					return false
+				}
+			}
+		case "class_declaration":
+			if body := firstNamedChildOfKind(n, map[string]struct{}{"class_body": {}}); body != nil {
+				found = body
+				return false
+			}
+		}
+		return true
+	})
+	return found
+}
+
+// containerSpec names a declaration node kind and the body node kind
+// whose named children are the split points.
+type containerSpec struct {
+	decl string
+	body string
+	// bodyField, when set, is the field name the body hangs off; when
+	// empty the body is found by scanning named children for `body`.
+	bodyField string
+}
+
+// braceContainer builds a findContainer that walks the AST for the
+// first declaration matching any of the specs and returns its body
+// node. Used by every brace-bodied grammar.
+func braceContainer(specs ...containerSpec) func(*sitter.Node) *sitter.Node {
+	byDecl := make(map[string]containerSpec, len(specs))
+	for _, s := range specs {
+		byDecl[s.decl] = s
+	}
+	return func(root *sitter.Node) *sitter.Node {
+		var found *sitter.Node
+		walkNodes(root, func(n *sitter.Node) bool {
+			if found != nil {
+				return false
+			}
+			spec, ok := byDecl[n.Type()]
+			if !ok {
+				return true
+			}
+			body := bodyOf(n, spec)
+			if body != nil {
+				found = body
+				return false
+			}
+			return true
+		})
+		return found
+	}
+}
+
+// bodyOf locates the body node of a declaration per its containerSpec.
+func bodyOf(decl *sitter.Node, spec containerSpec) *sitter.Node {
+	if spec.bodyField != "" {
+		body := decl.ChildByFieldName(spec.bodyField)
+		if body != nil && body.Type() == spec.body {
+			return body
+		}
+	}
+	n := int(decl.NamedChildCount())
+	for i := 0; i < n; i++ {
+		c := decl.NamedChild(i)
+		if c != nil && c.Type() == spec.body {
+			return c
+		}
+	}
+	return nil
+}
+
+// tsLikeContainer finds the splittable container in TypeScript /
+// JavaScript / JSX / TSX: a function/method body `statement_block`, an
+// arrow function's block body, or a class body / interface body.
+func tsLikeContainer(root *sitter.Node) *sitter.Node {
+	decls := map[string]struct{}{
+		"function_declaration":           {},
+		"generator_function_declaration": {},
+		"method_definition":              {},
+		"arrow_function":                 {},
+		"function_expression":            {},
+	}
+	bodyKinds := map[string]struct{}{
+		"statement_block": {},
+		"class_body":      {},
+	}
+	var found *sitter.Node
+	walkNodes(root, func(n *sitter.Node) bool {
+		if found != nil {
+			return false
+		}
+		t := n.Type()
+		if t == "class_declaration" || t == "class" || t == "interface_declaration" {
+			if body := firstNamedChildOfKind(n, bodyKinds); body != nil {
+				found = body
+				return false
+			}
+		}
+		if _, ok := decls[t]; ok {
+			if body := n.ChildByFieldName("body"); body != nil {
+				if _, ok := bodyKinds[body.Type()]; ok {
+					found = body
+					return false
+				}
+			}
+		}
+		return true
+	})
+	return found
+}
+
+// pythonContainer finds the `block` body of the first function or
+// class definition in a parsed Python span.
+func pythonContainer(root *sitter.Node) *sitter.Node {
+	decls := map[string]struct{}{
+		"function_definition": {},
+		"class_definition":    {},
+	}
+	var found *sitter.Node
+	walkNodes(root, func(n *sitter.Node) bool {
+		if found != nil {
+			return false
+		}
+		if _, ok := decls[n.Type()]; ok {
+			if body := n.ChildByFieldName("body"); body != nil && body.Type() == "block" {
+				found = body
+				return false
+			}
+		}
+		return true
+	})
+	return found
+}
+
+// firstNamedChildOfKind returns the first named child whose kind is in
+// the allowlist, or nil.
+func firstNamedChildOfKind(n *sitter.Node, kinds map[string]struct{}) *sitter.Node {
+	cnt := int(n.NamedChildCount())
+	for i := 0; i < cnt; i++ {
+		c := n.NamedChild(i)
+		if c == nil {
+			continue
+		}
+		if _, ok := kinds[c.Type()]; ok {
+			return c
+		}
+	}
+	return nil
+}
+
+// walkNodes does a pre-order DFS over the tree-sitter tree, calling
+// visit on each node. visit returns false to prune the subtree.
+func walkNodes(n *sitter.Node, visit func(*sitter.Node) bool) {
+	if n == nil {
+		return
+	}
+	if !visit(n) {
+		return
+	}
+	cnt := int(n.NamedChildCount())
+	for i := 0; i < cnt; i++ {
+		walkNodes(n.NamedChild(i), visit)
+	}
+}

internal/embedding/chunk_test.go

@@ -0,0 +1,276 @@
+package embedding
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestChunkSymbol_ShortFunctionIsOneChunk asserts a function below the
+// line threshold is embedded whole — one chunk, no splitting.
+func TestChunkSymbol_ShortFunctionIsOneChunk(t *testing.T) {
+	src := `func add(a, b int) int {
+	return a + b
+}`
+	chunks := ChunkSymbol([]byte(src), "go", "x.go::add", ChunkOptions{ThresholdLines: 60, WindowLines: 40})
+	require.Len(t, chunks, 1, "a short function must produce exactly one chunk")
+	assert.Equal(t, src, chunks[0].Text)
+	assert.Equal(t, "x.go::add", chunks[0].ParentID)
+	assert.Equal(t, 0, chunks[0].WindowIndex)
+}
+
+// TestChunkSymbol_LongGoFuncSplitsOnBlocks asserts a long Go function
+// is split into multiple windows on its top-level statements, that
+// every chunk carries the parent ID and a sequential WindowIndex, and
+// that the rejoined windows reproduce the original source exactly.
+func TestChunkSymbol_LongGoFuncSplitsOnBlocks(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("func bigFunc() {\n")
+	// 40 single-line statements — well past a 10-line window cap, so
+	// the splitter must produce several windows.
+	for i := 0; i < 40; i++ {
+		b.WriteString("\tx := compute()\n")
+	}
+	b.WriteString("}")
+	src := b.String()
+
+	chunks := ChunkSymbol([]byte(src), "go", "x.go::bigFunc", ChunkOptions{ThresholdLines: 10, WindowLines: 10})
+	require.Greater(t, len(chunks), 1, "a long function must split into more than one window")
+
+	rejoined := strings.Builder{}
+	for i, c := range chunks {
+		assert.Equal(t, "x.go::bigFunc", c.ParentID, "every chunk carries the parent ID")
+		assert.Equal(t, i, c.WindowIndex, "WindowIndex must be sequential from 0")
+		rejoined.WriteString(c.Text)
+	}
+	assert.Equal(t, src, rejoined.String(),
+		"concatenating the windows must reproduce the symbol source byte-for-byte")
+}
+
+// TestChunkSymbol_WindowCapRespected asserts no window (except one
+// containing a single oversized statement) exceeds the line cap.
+func TestChunkSymbol_WindowCapRespected(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("func paged() {\n")
+	for i := 0; i < 60; i++ {
+		b.WriteString("\tstep()\n")
+	}
+	b.WriteString("}")
+	src := b.String()
+
+	const window = 12
+	chunks := ChunkSymbol([]byte(src), "go", "x.go::paged", ChunkOptions{ThresholdLines: 20, WindowLines: window})
+	require.Greater(t, len(chunks), 1)
+	for i, c := range chunks {
+		lines := strings.Count(c.Text, "\n") + 1
+		// Allow generous slack — the first window also carries the
+		// signature line and the last the closing brace; the invariant
+		// is that windows are bounded, not exact.
+		assert.LessOrEqual(t, lines, window+4,
+			"window %d has %d lines, expected near the %d cap", i, lines, window)
+	}
+}
+
+// TestChunkSymbol_LongTypeSplitsOnFields asserts a large Go struct is
+// split on its field declarations.
+func TestChunkSymbol_LongTypeSplitsOnFields(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("type BigStruct struct {\n")
+	for i := 0; i < 50; i++ {
+		b.WriteString("\tFieldX int\n")
+	}
+	b.WriteString("}")
+	src := b.String()
+
+	chunks := ChunkSymbol([]byte(src), "go", "x.go::BigStruct", ChunkOptions{ThresholdLines: 15, WindowLines: 12})
+	require.Greater(t, len(chunks), 1, "a large struct must split on its fields")
+	rejoined := strings.Builder{}
+	for _, c := range chunks {
+		rejoined.WriteString(c.Text)
+	}
+	assert.Equal(t, src, rejoined.String())
+}
+
+// TestChunkSymbol_UnknownLanguageWhole asserts a language with no
+// splitter yields a single whole-symbol chunk regardless of size.
+func TestChunkSymbol_UnknownLanguageWhole(t *testing.T) {
+	src := strings.Repeat("line of cobol\n", 200)
+	chunks := ChunkSymbol([]byte(src), "cobol", "x.cob::Thing", ChunkOptions{ThresholdLines: 10, WindowLines: 5})
+	require.Len(t, chunks, 1, "no splitter for the language → embed whole")
+	assert.Equal(t, src, chunks[0].Text)
+}
+
+// TestChunkSymbol_GarbageStillOneChunk asserts a span that fails to
+// parse falls back to a single chunk rather than erroring or dropping
+// the symbol.
+func TestChunkSymbol_GarbageStillOneChunk(t *testing.T) {
+	src := strings.Repeat("}{ ][ <<>> ;;;\n", 100)
+	chunks := ChunkSymbol([]byte(src), "go", "x.go::junk", ChunkOptions{ThresholdLines: 10, WindowLines: 5})
+	require.GreaterOrEqual(t, len(chunks), 1, "a parse failure must still yield at least one chunk")
+	// Whatever the split, the windows must still cover the whole input.
+	rejoined := strings.Builder{}
+	for _, c := range chunks {
+		rejoined.WriteString(c.Text)
+	}
+	assert.Equal(t, src, rejoined.String())
+}
+
+// TestChunkSymbol_EmptyInput asserts the empty-span edge case yields a
+// single empty chunk.
+func TestChunkSymbol_EmptyInput(t *testing.T) {
+	chunks := ChunkSymbol(nil, "go", "x.go::empty", ChunkOptions{})
+	require.Len(t, chunks, 1)
+	assert.Equal(t, "", chunks[0].Text)
+}
+
+// TestChunkSymbol_DefaultsApplied asserts zero-valued options fall back
+// to the package defaults: a function under DefaultChunkThresholdLines
+// stays whole.
+func TestChunkSymbol_DefaultsApplied(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("func midsize() {\n")
+	for i := 0; i < DefaultChunkThresholdLines-5; i++ {
+		b.WriteString("\tdoThing()\n")
+	}
+	b.WriteString("}")
+	chunks := ChunkSymbol([]byte(b.String()), "go", "x.go::midsize", ChunkOptions{})
+	assert.Len(t, chunks, 1, "a function under the default threshold must stay whole")
+}
+
+// TestChunkSymbol_LongPythonFuncSplits exercises the indent-language
+// path: a long Python function splits on its block statements.
+func TestChunkSymbol_LongPythonFuncSplits(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("def big():\n")
+	for i := 0; i < 40; i++ {
+		b.WriteString("    x = compute()\n")
+	}
+	src := b.String()
+	chunks := ChunkSymbol([]byte(src), "python", "x.py::big", ChunkOptions{ThresholdLines: 10, WindowLines: 10})
+	require.Greater(t, len(chunks), 1, "a long Python function must split")
+	rejoined := strings.Builder{}
+	for _, c := range chunks {
+		rejoined.WriteString(c.Text)
+	}
+	assert.Equal(t, src, rejoined.String())
+}
+
+// assertSplitsAndRejoins is the shared body for the new-language
+// sub-chunking tests: a large symbol must split into more than one
+// window, every chunk carries the parent ID and a sequential
+// WindowIndex, and the rejoined windows reproduce the source exactly.
+func assertSplitsAndRejoins(t *testing.T, src, lang, parentID string) {
+	t.Helper()
+	chunks := ChunkSymbol([]byte(src), lang, parentID, ChunkOptions{ThresholdLines: 10, WindowLines: 10})
+	require.Greaterf(t, len(chunks), 1, "a long %s symbol must split into more than one window", lang)
+	rejoined := strings.Builder{}
+	for i, c := range chunks {
+		assert.Equal(t, parentID, c.ParentID, "every chunk carries the parent ID")
+		assert.Equal(t, i, c.WindowIndex, "WindowIndex must be sequential from 0")
+		rejoined.WriteString(c.Text)
+	}
+	assert.Equalf(t, src, rejoined.String(),
+		"concatenating the %s windows must reproduce the symbol source byte-for-byte", lang)
+}
+
+// TestChunkSymbol_LongRubyMethodSplits exercises the Ruby splitter: a
+// long method's body_statement children are the split points.
+func TestChunkSymbol_LongRubyMethodSplits(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("def big(x)\n")
+	for i := 0; i < 40; i++ {
+		b.WriteString("  y = x + 1\n")
+	}
+	b.WriteString("end")
+	assertSplitsAndRejoins(t, b.String(), "ruby", "x.rb::big")
+}
+
+// TestChunkSymbol_LongRubyClassSplits asserts a large Ruby class splits
+// on its body_statement members.
+func TestChunkSymbol_LongRubyClassSplits(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("class Big\n")
+	for i := 0; i < 40; i++ {
+		b.WriteString("  def m; 1; end\n")
+	}
+	b.WriteString("end")
+	assertSplitsAndRejoins(t, b.String(), "ruby", "x.rb::Big")
+}
+
+// TestChunkSymbol_LongPHPMethodSplits exercises the PHP splitter: a long
+// function's compound_statement children are the split points.
+func TestChunkSymbol_LongPHPMethodSplits(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("<?php\nfunction big($x) {\n")
+	for i := 0; i < 40; i++ {
+		b.WriteString("  $y = $x + 1;\n")
+	}
+	b.WriteString("}")
+	assertSplitsAndRejoins(t, b.String(), "php", "x.php::big")
+}
+
+// TestChunkSymbol_LongKotlinFuncSplits exercises the Kotlin splitter: a
+// function's function_body -> statements children are the split points.
+func TestChunkSymbol_LongKotlinFuncSplits(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("fun big(x: Int): Int {\n")
+	for i := 0; i < 40; i++ {
+		b.WriteString("  val y = x + 1\n")
+	}
+	b.WriteString("  return x\n}")
+	assertSplitsAndRejoins(t, b.String(), "kotlin", "x.kt::big")
+}
+
+// TestChunkSymbol_LongKotlinClassSplits asserts a large Kotlin class
+// splits on its class_body members.
+func TestChunkSymbol_LongKotlinClassSplits(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("class Big {\n")
+	for i := 0; i < 40; i++ {
+		b.WriteString("  fun m(): Int { return 1 }\n")
+	}
+	b.WriteString("}")
+	assertSplitsAndRejoins(t, b.String(), "kotlin", "x.kt::Big")
+}
+
+// TestChunkSymbol_LongSwiftFuncSplits exercises the Swift splitter: a
+// function's function_body -> statements children are the split points.
+func TestChunkSymbol_LongSwiftFuncSplits(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("func big(x: Int) -> Int {\n")
+	for i := 0; i < 40; i++ {
+		b.WriteString("  let y = x + 1\n")
+	}
+	b.WriteString("  return x\n}")
+	assertSplitsAndRejoins(t, b.String(), "swift", "x.swift::big")
+}
+
+// TestChunkSymbol_LongSwiftClassSplits asserts a large Swift class
+// splits on its class_body members.
+func TestChunkSymbol_LongSwiftClassSplits(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("class Big {\n")
+	for i := 0; i < 40; i++ {
+		b.WriteString("  func m() -> Int { return 1 }\n")
+	}
+	b.WriteString("}")
+	assertSplitsAndRejoins(t, b.String(), "swift", "x.swift::Big")
+}
+
+// TestChunkSymbol_NewLanguagesFailSoft asserts the new-language
+// splitters preserve the exact fail-soft contract: a span that fails to
+// parse still yields windows whose rejoin reproduces the input.
+func TestChunkSymbol_NewLanguagesFailSoft(t *testing.T) {
+	garbage := strings.Repeat("}{ ][ <<>> ;;;\n", 100)
+	for _, lang := range []string{"ruby", "php", "kotlin", "swift"} {
+		chunks := ChunkSymbol([]byte(garbage), lang, "x::junk", ChunkOptions{ThresholdLines: 10, WindowLines: 5})
+		require.GreaterOrEqualf(t, len(chunks), 1, "%s parse failure must still yield a chunk", lang)
+		rejoined := strings.Builder{}
+		for _, c := range chunks {
+			rejoined.WriteString(c.Text)
+		}
+		assert.Equalf(t, garbage, rejoined.String(), "%s windows must cover the whole input", lang)
+	}
+}

internal/embedding/gomlx.go

@@ -0,0 +1,121 @@
+//go:build embeddings_gomlx
+
+package embedding
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/knights-analytics/hugot"
+	"github.com/knights-analytics/hugot/pipelines"
+
+	"github.com/zzet/gortex/internal/platform"
+)
+
+const gomlxModelName = "sentence-transformers/all-MiniLM-L6-v2"
+
+// GoMLXProvider uses Hugot with the XLA/GoMLX backend for transformer embeddings.
+// XLA/PJRT plugin auto-downloads on first use (~100MB).
+type GoMLXProvider struct {
+	session   *hugot.Session
+	pipeline  *pipelines.FeatureExtractionPipeline
+	dims      int
+	truncator *tokenTruncator
+	mu        sync.Mutex
+}
+
+func newGoMLXProvider() (Provider, error) {
+	session, err := hugot.NewXLASession(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("gomlx/xla session: %w", err)
+	}
+
+	modelPath, err := ensureGoMLXModel()
+	if err != nil {
+		_ = session.Destroy()
+		return nil, fmt.Errorf("gomlx model: %w", err)
+	}
+
+	config := hugot.FeatureExtractionConfig{
+		ModelPath: modelPath,
+		Name:      "gortex-embeddings-gomlx",
+		Options: []hugot.FeatureExtractionOption{
+			pipelines.WithNormalization(),
+		},
+	}
+
+	pipeline, err := hugot.NewPipeline(session, config)
+	if err != nil {
+		_ = session.Destroy()
+		return nil, fmt.Errorf("gomlx pipeline: %w", err)
+	}
+
+	// Belt-and-suspenders token truncation: the XLA path's Rust tokenizer
+	// already truncates, but keeping the client-side cap here matches the
+	// pure-Go provider and covers a degraded tokenizer. See hugot.go.
+	truncator, terr := newTokenTruncator(modelPath)
+	if terr != nil {
+		fmt.Fprintf(os.Stderr, "[gortex embedding] %v\n", terr)
+	}
+
+	return &GoMLXProvider{
+		session:   session,
+		pipeline:  pipeline,
+		dims:      384,
+		truncator: truncator,
+	}, nil
+}
+
+func (p *GoMLXProvider) Embed(ctx context.Context, text string) ([]float32, error) {
+	vecs, err := p.EmbedBatch(ctx, []string{text})
+	if err != nil {
+		return nil, err
+	}
+	if len(vecs) == 0 {
+		return nil, fmt.Errorf("gomlx returned no embeddings")
+	}
+	return vecs[0], nil
+}
+
+func (p *GoMLXProvider) EmbedBatch(ctx context.Context, texts []string) ([][]float32, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	texts = p.truncator.TruncateAll(texts)
+
+	output, err := p.pipeline.RunPipeline(ctx, texts)
+	if err != nil {
+		return nil, fmt.Errorf("gomlx run: %w", err)
+	}
+	if err := validateBatch("gomlx", texts, output.Embeddings, p.dims); err != nil {
+		return nil, err
+	}
+	return output.Embeddings, nil
+}
+
+func (p *GoMLXProvider) Dimensions() int { return p.dims }
+
+func (p *GoMLXProvider) Close() error {
+	if p.session != nil {
+		return p.session.Destroy()
+	}
+	return nil
+}
+
+func ensureGoMLXModel() (string, error) {
+	dest := platform.ModelsDir()
+	modelDir := filepath.Join(dest, "sentence-transformers_all-MiniLM-L6-v2")
+
+	if _, err := os.Stat(filepath.Join(modelDir, "tokenizer.json")); err == nil {
+		return modelDir, nil
+	}
+
+	path, err := hugot.DownloadModel(context.Background(), gomlxModelName, dest, hugot.NewDownloadOptions())
+	if err != nil {
+		return "", fmt.Errorf("download model: %w", err)
+	}
+	return path, nil
+}

internal/embedding/gomlx_stub.go

@@ -0,0 +1,9 @@
+//go:build !embeddings_gomlx
+
+package embedding
+
+import "fmt"
+
+func newGoMLXProvider() (Provider, error) {
+	return nil, fmt.Errorf("GoMLX provider not compiled in (build with -tags \"embeddings_gomlx XLA\"): %w", ErrBackendNotCompiled)
+}

internal/embedding/hugot.go

@@ -0,0 +1,269 @@
+package embedding
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"sync"
+
+	"github.com/knights-analytics/hugot"
+	"github.com/knights-analytics/hugot/pipelines"
+
+	"github.com/zzet/gortex/internal/platform"
+)
+
+// miniLMRepo is the sentence-transformers MiniLM-L6-v2 repo used by
+// the default bundled variants. Other variants (code-tuned, multilingual,
+// …) carry their own RepoID on the variant spec.
+const miniLMRepo = "sentence-transformers/all-MiniLM-L6-v2"
+
+// HugotProvider uses Hugot with the pure Go backend for offline transformer embeddings.
+// Model auto-downloads from Hugging Face on first use.
+type HugotProvider struct {
+	session   *hugot.Session
+	pipeline  *pipelines.FeatureExtractionPipeline
+	dims      int
+	truncator *tokenTruncator
+	mu        sync.Mutex
+}
+
+// DefaultHugotVariant is the short name of the variant newHugotProvider
+// loads when no explicit choice is made. MiniLM-L6-v2 fp32 — baseline
+// quality, slowest inference, small footprint.
+const DefaultHugotVariant = "fp32"
+
+// HugotVariant describes one embeddable model: which HuggingFace repo
+// to pull, which ONNX variant file inside it to load, the embedding
+// dimension, and a human-readable label. Exposed so `gortex eval
+// embedders` can enumerate + compare arbitrary models.
+type HugotVariant struct {
+	RepoID     string // HuggingFace repo path, e.g. "BAAI/bge-code-v1"
+	OnnxFile   string // path inside the repo, e.g. "onnx/model.onnx"
+	Dimensions int    // embedding dim (must match model output)
+	Label      string // short human label shown in the report
+}
+
+var hugotVariants = map[string]HugotVariant{
+	// MiniLM-L6-v2 variants — general-English baseline.
+	"fp32":         {RepoID: miniLMRepo, OnnxFile: "onnx/model.onnx", Dimensions: 384, Label: "MiniLM-L6 fp32"},
+	"o2":           {RepoID: miniLMRepo, OnnxFile: "onnx/model_O2.onnx", Dimensions: 384, Label: "MiniLM-L6 fp32-O2"},
+	"o3":           {RepoID: miniLMRepo, OnnxFile: "onnx/model_O3.onnx", Dimensions: 384, Label: "MiniLM-L6 fp32-O3"},
+	"o4":           {RepoID: miniLMRepo, OnnxFile: "onnx/model_O4.onnx", Dimensions: 384, Label: "MiniLM-L6 fp32-O4"},
+	"qint8_arm64":  {RepoID: miniLMRepo, OnnxFile: "onnx/model_qint8_arm64.onnx", Dimensions: 384, Label: "MiniLM-L6 qint8-arm64"},
+	"qint8_avx512": {RepoID: miniLMRepo, OnnxFile: "onnx/model_qint8_avx512.onnx", Dimensions: 384, Label: "MiniLM-L6 qint8-avx512"},
+	"quint8_avx2":  {RepoID: miniLMRepo, OnnxFile: "onnx/model_quint8_avx2.onnx", Dimensions: 384, Label: "MiniLM-L6 quint8-avx2"},
+
+	// General retrieval-tuned models — trained for search (not just
+	// sentence similarity), published with ONNX exports, drop-in under
+	// Hugot's pure-Go runtime.
+	"bge_small": {RepoID: "BAAI/bge-small-en-v1.5", OnnxFile: "onnx/model.onnx", Dimensions: 384, Label: "BGE small-en-v1.5"},
+
+	// Code-tuned models — hypothesis: trained on code, should beat
+	// MiniLM on concept / multi-hop code queries.
+	"jina_code": {RepoID: "jinaai/jina-embeddings-v2-base-code", OnnxFile: "onnx/model.onnx", Dimensions: 768, Label: "Jina v2 base-code"},
+
+	// Not currently loadable under the pure-Go runtime — ships as
+	// safetensors only on HuggingFace. Kept here so `gortex eval
+	// embedders` surfaces a clear error rather than silently falling
+	// back. To use: export locally via optimum-cli + `--local-model`
+	// (follow-up), or route via APIProvider against an OpenAI-compat
+	// embeddings endpoint.
+	"bge_code": {RepoID: "BAAI/bge-code-v1", OnnxFile: "onnx/model.onnx", Dimensions: 1024, Label: "BAAI bge-code-v1 (no ONNX)"},
+}
+
+// LookupHugotVariant returns the variant spec for a short name, or false.
+func LookupHugotVariant(name string) (HugotVariant, bool) {
+	v, ok := hugotVariants[name]
+	return v, ok
+}
+
+// KnownHugotVariants returns sorted short names of every known variant.
+func KnownHugotVariants() []string {
+	names := make([]string, 0, len(hugotVariants))
+	for k := range hugotVariants {
+		names = append(names, k)
+	}
+	sort.Strings(names)
+	return names
+}
+
+func newHugotProvider() (Provider, error) {
+	spec, _ := LookupHugotVariant(DefaultHugotVariant)
+	return newHugotProviderWithSpec(spec)
+}
+
+func newHugotProviderWithSpec(spec HugotVariant) (Provider, error) {
+	if spec.RepoID == "" || spec.OnnxFile == "" {
+		return nil, fmt.Errorf("invalid variant spec: RepoID=%q OnnxFile=%q", spec.RepoID, spec.OnnxFile)
+	}
+	session, err := hugot.NewGoSession(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("hugot session: %w", err)
+	}
+
+	modelPath, err := ensureHugotModel(spec)
+	if err != nil {
+		_ = session.Destroy()
+		return nil, fmt.Errorf("hugot model: %w", err)
+	}
+
+	config := hugot.FeatureExtractionConfig{
+		ModelPath:    modelPath,
+		Name:         "gortex-embeddings",
+		OnnxFilename: filepath.Base(spec.OnnxFile),
+		Options: []hugot.FeatureExtractionOption{
+			pipelines.WithNormalization(),
+		},
+	}
+
+	pipeline, err := hugot.NewPipeline(session, config)
+	if err != nil {
+		_ = session.Destroy()
+		return nil, fmt.Errorf("hugot pipeline: %w", err)
+	}
+
+	dims := spec.Dimensions
+	if dims == 0 {
+		dims = 384 // conservative fallback
+	}
+
+	// Cap inputs at the model's positional budget before they reach the
+	// pipeline. The pure-Go tokenizer path does not honour
+	// max_position_embeddings, so an over-long text would otherwise abort the
+	// whole vector index with a tensor shape mismatch. A degraded truncator
+	// (missing config.json / corrupt tokenizer.json) still works via a rune
+	// clamp, so we warn and continue rather than fail the provider.
+	truncator, terr := newTokenTruncator(modelPath)
+	if terr != nil {
+		fmt.Fprintf(os.Stderr, "[gortex embedding] %v\n", terr)
+	}
+
+	return &HugotProvider{
+		session:   session,
+		pipeline:  pipeline,
+		dims:      dims,
+		truncator: truncator,
+	}, nil
+}
+
+func (p *HugotProvider) Embed(ctx context.Context, text string) ([]float32, error) {
+	vecs, err := p.EmbedBatch(ctx, []string{text})
+	if err != nil {
+		return nil, err
+	}
+	if len(vecs) == 0 {
+		return nil, fmt.Errorf("hugot returned no embeddings")
+	}
+	return vecs[0], nil
+}
+
+func (p *HugotProvider) EmbedBatch(ctx context.Context, texts []string) ([][]float32, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	texts = p.truncator.TruncateAll(texts)
+
+	output, err := p.pipeline.RunPipeline(ctx, texts)
+	if err != nil {
+		return nil, fmt.Errorf("hugot run: %w", err)
+	}
+	if err := validateBatch("hugot", texts, output.Embeddings, p.dims); err != nil {
+		return nil, err
+	}
+	return output.Embeddings, nil
+}
+
+func (p *HugotProvider) Dimensions() int { return p.dims }
+
+func (p *HugotProvider) Close() error {
+	if p.session != nil {
+		return p.session.Destroy()
+	}
+	return nil
+}
+
+// embeddingOfflineEnv, when set to a truthy value (1/true), disables
+// network downloads of embedding models. ensureHugotModel then returns
+// an error for any not-yet-cached model so NewLocalProvider falls back
+// to the static provider. Useful for air-gapped hosts, sandboxes, and
+// tests that must not touch the network.
+const embeddingOfflineEnv = "GORTEX_EMBEDDING_OFFLINE"
+
+// embeddingDownloadsDisabled reports whether model downloads are turned
+// off via embeddingOfflineEnv. An unset or falsey value keeps the
+// default behaviour (download on first use).
+func embeddingDownloadsDisabled() bool {
+	on, _ := strconv.ParseBool(os.Getenv(embeddingOfflineEnv))
+	return on
+}
+
+// ensureHugotModel downloads the variant's HuggingFace repo if needed
+// and returns the on-disk path Hugot will load from. The ONNX file is
+// specified via DownloadOptions because most repos ship multiple
+// variants and the downloader refuses to guess. The cache layout
+// mirrors Hugot's own convention: `<cache>/<org>_<model-name>/…`.
+func ensureHugotModel(spec HugotVariant) (string, error) {
+	dest := platform.ModelsDir()
+	modelDir := filepath.Join(dest, hfCacheDirName(spec.RepoID))
+
+	tokenizerReady := false
+	if _, err := os.Stat(filepath.Join(modelDir, "tokenizer.json")); err == nil {
+		tokenizerReady = true
+	}
+	// The downloader flattens `<subdir>/<file>.onnx` to `<file>.onnx`
+	// in modelDir, so check both the nested path and the basename.
+	variantReady := false
+	if _, err := os.Stat(filepath.Join(modelDir, spec.OnnxFile)); err == nil {
+		variantReady = true
+	} else if _, err := os.Stat(filepath.Join(modelDir, filepath.Base(spec.OnnxFile))); err == nil {
+		variantReady = true
+	}
+	if tokenizerReady && variantReady {
+		return modelDir, nil
+	}
+
+	// Offline guard: when downloads are disabled, refuse to fetch a
+	// missing model instead of blocking on (or racing inside) the
+	// network downloader. NewLocalProvider treats this error as "backend
+	// unavailable" and falls through to the static provider.
+	if embeddingDownloadsDisabled() {
+		return "", fmt.Errorf("embedding model %s (%s) not cached and downloads are disabled via %s", spec.RepoID, spec.OnnxFile, embeddingOfflineEnv)
+	}
+
+	opts := hugot.NewDownloadOptions()
+	opts.OnnxFilePath = spec.OnnxFile
+	path, err := hugot.DownloadModel(context.Background(), spec.RepoID, dest, opts)
+	if err != nil {
+		return "", fmt.Errorf("download %s (%s): %w", spec.RepoID, spec.OnnxFile, err)
+	}
+	return path, nil
+}
+
+// hfCacheDirName turns a HuggingFace repo path ("org/name") into the
+// directory name Hugot writes to ("org_name"). Hugot already does this
+// internally when downloading; we mirror the convention so the
+// tokenizer/variant existence checks find the cached files.
+func hfCacheDirName(repoID string) string {
+	// Use path separator normalisation rather than a raw replace so
+	// nested subdirs in custom repos don't corrupt the cache layout.
+	return filepath.Clean(filepath.FromSlash(
+		replaceAllSlashes(repoID, "_"),
+	))
+}
+
+// replaceAllSlashes is a tiny helper to avoid pulling in strings just
+// for one call site.
+func replaceAllSlashes(s, repl string) string {
+	out := make([]byte, 0, len(s))
+	for _, r := range s {
+		if r == '/' {
+			out = append(out, repl...)
+		} else {
+			out = append(out, string(r)...)
+		}
+	}
+	return string(out)
+}

internal/embedding/nop.go

@@ -0,0 +1,23 @@
+package embedding
+
+import (
+	"context"
+	"errors"
+)
+
+// ErrDisabled is returned when embeddings are not enabled.
+var ErrDisabled = errors.New("embeddings disabled")
+
+// NopProvider is a no-op embedding provider used when embeddings are disabled.
+type NopProvider struct{}
+
+func (NopProvider) Embed(_ context.Context, _ string) ([]float32, error) {
+	return nil, ErrDisabled
+}
+
+func (NopProvider) EmbedBatch(_ context.Context, _ []string) ([][]float32, error) {
+	return nil, ErrDisabled
+}
+
+func (NopProvider) Dimensions() int { return 0 }
+func (NopProvider) Close() error    { return nil }

internal/embedding/onnx.go

@@ -0,0 +1,378 @@
+//go:build embeddings_onnx
+
+package embedding
+
+import (
+	"bufio"
+	"compress/gzip"
+	"context"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+
+	ort "github.com/yalue/onnxruntime_go"
+
+	"github.com/zzet/gortex/internal/platform"
+)
+
+const (
+	onnxMaxSeqLen = 128
+	onnxDims      = 384
+	clsTokenID    = 101
+	sepTokenID    = 102
+	unkTokenID    = 100
+	padTokenID    = 0
+)
+
+// ONNXProvider uses GTE-small via ONNX Runtime for high-quality embeddings.
+// Creates a single session with fixed-size input tensors for fast reuse.
+type ONNXProvider struct {
+	vocab   map[string]int64
+	session *ort.AdvancedSession
+
+	// Pre-allocated tensors (fixed shape: 1 × onnxMaxSeqLen).
+	inputIDs      *ort.Tensor[int64]
+	attentionMask *ort.Tensor[int64]
+	tokenTypeIDs  *ort.Tensor[int64]
+	output        *ort.Tensor[float32]
+
+	mu sync.Mutex
+}
+
+func newONNXProvider() (Provider, error) {
+	modelDir := findONNXModelDir()
+	if modelDir == "" {
+		return nil, fmt.Errorf("ONNX model not found; this backend never auto-downloads — manually place model.onnx + vocab.txt in ~/.gortex/models/gte-small/ (plus `brew install onnxruntime` or the distro equivalent); see docs/semantic-search.md")
+	}
+
+	modelPath := filepath.Join(modelDir, "model.onnx")
+	vocabPath := filepath.Join(modelDir, "vocab.txt")
+
+	if _, err := os.Stat(modelPath); err != nil {
+		return nil, fmt.Errorf("model.onnx not found in %s", modelDir)
+	}
+
+	vocab, err := loadVocab(vocabPath)
+	if err != nil {
+		return nil, fmt.Errorf("load vocab: %w", err)
+	}
+
+	libPath := findONNXRuntimeLib()
+	if libPath == "" {
+		return nil, fmt.Errorf("libonnxruntime not found; install via: brew install onnxruntime")
+	}
+	ort.SetSharedLibraryPath(libPath)
+	if err := ort.InitializeEnvironment(); err != nil {
+		return nil, fmt.Errorf("ONNX Runtime init: %w", err)
+	}
+
+	// Pre-allocate fixed-size tensors.
+	shape := ort.Shape{1, onnxMaxSeqLen}
+	outputShape := ort.Shape{1, onnxMaxSeqLen, onnxDims}
+
+	inputIDs, err := ort.NewEmptyTensor[int64](shape)
+	if err != nil {
+		return nil, fmt.Errorf("input_ids tensor: %w", err)
+	}
+	attMask, err := ort.NewEmptyTensor[int64](shape)
+	if err != nil {
+		return nil, fmt.Errorf("attention_mask tensor: %w", err)
+	}
+	tokenTypes, err := ort.NewEmptyTensor[int64](shape)
+	if err != nil {
+		return nil, fmt.Errorf("token_type_ids tensor: %w", err)
+	}
+	output, err := ort.NewEmptyTensor[float32](outputShape)
+	if err != nil {
+		return nil, fmt.Errorf("output tensor: %w", err)
+	}
+
+	// Create session once with fixed shapes.
+	session, err := ort.NewAdvancedSession(modelPath,
+		[]string{"input_ids", "attention_mask", "token_type_ids"},
+		[]string{"last_hidden_state"},
+		[]ort.ArbitraryTensor{inputIDs, attMask, tokenTypes},
+		[]ort.ArbitraryTensor{output},
+		nil,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("ONNX session: %w", err)
+	}
+
+	return &ONNXProvider{
+		vocab:         vocab,
+		session:       session,
+		inputIDs:      inputIDs,
+		attentionMask: attMask,
+		tokenTypeIDs:  tokenTypes,
+		output:        output,
+	}, nil
+}
+
+func (p *ONNXProvider) Embed(_ context.Context, text string) ([]float32, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.embedLocked(text)
+}
+
+func (p *ONNXProvider) EmbedBatch(_ context.Context, texts []string) ([][]float32, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	results := make([][]float32, len(texts))
+	for i, text := range texts {
+		vec, err := p.embedLocked(text)
+		if err != nil {
+			return nil, fmt.Errorf("embed text %d: %w", i, err)
+		}
+		results[i] = vec
+	}
+	return results, nil
+}
+
+func (p *ONNXProvider) Dimensions() int { return onnxDims }
+
+func (p *ONNXProvider) Close() error {
+	if p.session != nil {
+		_ = p.session.Destroy()
+	}
+	if p.inputIDs != nil {
+		_ = p.inputIDs.Destroy()
+	}
+	if p.attentionMask != nil {
+		_ = p.attentionMask.Destroy()
+	}
+	if p.tokenTypeIDs != nil {
+		_ = p.tokenTypeIDs.Destroy()
+	}
+	if p.output != nil {
+		_ = p.output.Destroy()
+	}
+	return ort.DestroyEnvironment()
+}
+
+func (p *ONNXProvider) embedLocked(text string) ([]float32, error) {
+	// Tokenize and pad to fixed length.
+	tokenIDs := p.tokenize(text)
+
+	// Fill pre-allocated input tensors.
+	inputData := p.inputIDs.GetData()
+	attData := p.attentionMask.GetData()
+	ttData := p.tokenTypeIDs.GetData()
+
+	realTokens := 0
+	for i := 0; i < onnxMaxSeqLen; i++ {
+		if i < len(tokenIDs) {
+			inputData[i] = tokenIDs[i]
+			attData[i] = 1
+			realTokens++
+		} else {
+			inputData[i] = padTokenID
+			attData[i] = 0
+		}
+		ttData[i] = 0
+	}
+
+	// Run inference (session reused).
+	if err := p.session.Run(); err != nil {
+		return nil, fmt.Errorf("inference: %w", err)
+	}
+
+	// Mean pooling over non-padding tokens.
+	outputData := p.output.GetData()
+	embedding := make([]float32, onnxDims)
+	for i := 0; i < realTokens; i++ {
+		for j := 0; j < onnxDims; j++ {
+			embedding[j] += outputData[i*onnxDims+j]
+		}
+	}
+	if realTokens > 0 {
+		for j := range embedding {
+			embedding[j] /= float32(realTokens)
+		}
+	}
+
+	// L2 normalize.
+	var norm float64
+	for _, v := range embedding {
+		norm += float64(v) * float64(v)
+	}
+	norm = math.Sqrt(norm)
+	if norm > 1e-10 {
+		for j := range embedding {
+			embedding[j] /= float32(norm)
+		}
+	}
+
+	return embedding, nil
+}
+
+// tokenize performs basic WordPiece tokenization, padded to onnxMaxSeqLen.
+func (p *ONNXProvider) tokenize(text string) []int64 {
+	text = strings.ToLower(text)
+
+	var words []string
+	var current strings.Builder
+	for _, r := range text {
+		if r == ' ' || r == '\t' || r == '\n' || r == '/' || r == '.' || r == ':' || r == '_' || r == '-' {
+			if current.Len() > 0 {
+				words = append(words, current.String())
+				current.Reset()
+			}
+		} else {
+			current.WriteRune(r)
+		}
+	}
+	if current.Len() > 0 {
+		words = append(words, current.String())
+	}
+
+	ids := []int64{clsTokenID}
+	for _, word := range words {
+		if len(ids) >= onnxMaxSeqLen-1 {
+			break
+		}
+		wordIDs := p.wordPieceTokenize(word)
+		for _, id := range wordIDs {
+			if len(ids) >= onnxMaxSeqLen-1 {
+				break
+			}
+			ids = append(ids, id)
+		}
+	}
+	ids = append(ids, sepTokenID)
+	return ids
+}
+
+func (p *ONNXProvider) wordPieceTokenize(word string) []int64 {
+	if id, ok := p.vocab[word]; ok {
+		return []int64{id}
+	}
+
+	var ids []int64
+	remaining := word
+	for len(remaining) > 0 {
+		prefix := remaining
+		found := false
+		for len(prefix) > 0 {
+			lookup := prefix
+			if len(ids) > 0 {
+				lookup = "##" + prefix
+			}
+			if id, ok := p.vocab[lookup]; ok {
+				ids = append(ids, id)
+				remaining = remaining[len(prefix):]
+				found = true
+				break
+			}
+			prefix = prefix[:len(prefix)-1]
+		}
+		if !found {
+			ids = append(ids, unkTokenID)
+			break
+		}
+	}
+	return ids
+}
+
+// --- helpers ---
+
+func findONNXModelDir() string {
+	candidates := []string{
+		filepath.Join(platform.ModelsDir(), "gte-small"),
+		"/tmp/gte-small",
+	}
+	for _, dir := range candidates {
+		if _, err := os.Stat(filepath.Join(dir, "model.onnx")); err == nil {
+			return dir
+		}
+	}
+	return ""
+}
+
+func findONNXRuntimeLib() string {
+	switch runtime.GOOS {
+	case "darwin":
+		for _, p := range []string{
+			"/opt/homebrew/lib/libonnxruntime.dylib",
+			"/usr/local/lib/libonnxruntime.dylib",
+		} {
+			if _, err := os.Stat(p); err == nil {
+				return p
+			}
+		}
+	case "linux":
+		for _, p := range []string{
+			"/usr/lib/libonnxruntime.so",
+			"/usr/local/lib/libonnxruntime.so",
+			"/usr/lib/x86_64-linux-gnu/libonnxruntime.so",
+		} {
+			if _, err := os.Stat(p); err == nil {
+				return p
+			}
+		}
+	}
+	return ""
+}
+
+func loadVocab(path string) (map[string]int64, error) {
+	if strings.HasSuffix(path, ".gz") {
+		return loadVocabGz(path)
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	vocab := make(map[string]int64, 32000)
+	scanner := bufio.NewScanner(f)
+	var id int64
+	for scanner.Scan() {
+		line := scanner.Text()
+		if parts := strings.SplitN(line, "\t", 2); len(parts) == 2 {
+			word := parts[0]
+			fmt.Sscanf(parts[1], "%d", &id)
+			vocab[word] = id
+		} else {
+			vocab[line] = id
+			id++
+		}
+	}
+	return vocab, scanner.Err()
+}
+
+func loadVocabGz(path string) (map[string]int64, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		return nil, err
+	}
+	defer gz.Close()
+
+	vocab := make(map[string]int64, 32000)
+	scanner := bufio.NewScanner(gz)
+	var id int64
+	for scanner.Scan() {
+		line := scanner.Text()
+		if parts := strings.SplitN(line, "\t", 2); len(parts) == 2 {
+			word := parts[0]
+			fmt.Sscanf(parts[1], "%d", &id)
+			vocab[word] = id
+		} else {
+			vocab[line] = id
+			id++
+		}
+	}
+	return vocab, scanner.Err()
+}

internal/embedding/onnx_stub.go

@@ -0,0 +1,9 @@
+//go:build !embeddings_onnx
+
+package embedding
+
+import "fmt"
+
+func newONNXProvider() (Provider, error) {
+	return nil, fmt.Errorf("ONNX provider not compiled in (build with -tags embeddings_onnx): %w", ErrBackendNotCompiled)
+}

internal/embedding/onnx_test.go

@@ -0,0 +1,68 @@
+//go:build embeddings_onnx
+
+package embedding
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestONNXProvider_Embed(t *testing.T) {
+	p, err := newONNXProvider()
+	if err != nil {
+		t.Skipf("ONNX provider not available: %v", err)
+	}
+	defer p.Close()
+
+	assert.Equal(t, onnxDims, p.Dimensions())
+
+	ctx := context.Background()
+
+	v1, err := p.Embed(ctx, "function ValidateToken internal/auth/service.go")
+	require.NoError(t, err)
+	assert.Len(t, v1, onnxDims)
+
+	v2, err := p.Embed(ctx, "function CheckAuthentication internal/auth/checker.go")
+	require.NoError(t, err)
+
+	v3, err := p.Embed(ctx, "function ParseJSON internal/parser/json.go")
+	require.NoError(t, err)
+
+	// Auth-related functions should be more similar to each other than to parsing.
+	sim12 := cosine(v1, v2)
+	sim13 := cosine(v1, v3)
+
+	t.Logf("ValidateToken vs CheckAuth: %.4f", sim12)
+	t.Logf("ValidateToken vs ParseJSON: %.4f", sim13)
+
+	assert.Greater(t, sim12, sim13, "auth functions should be more similar to each other than to parser")
+}
+
+func TestONNXProvider_EmbedBatch(t *testing.T) {
+	p, err := newONNXProvider()
+	if err != nil {
+		t.Skipf("ONNX provider not available: %v", err)
+	}
+	defer p.Close()
+
+	ctx := context.Background()
+	vecs, err := p.EmbedBatch(ctx, []string{
+		"function Foo internal/a.go",
+		"method Bar internal/b.go",
+	})
+	require.NoError(t, err)
+	assert.Len(t, vecs, 2)
+	assert.Len(t, vecs[0], onnxDims)
+	assert.Len(t, vecs[1], onnxDims)
+}
+
+func cosine(a, b []float32) float64 {
+	var dot float64
+	for i := range a {
+		dot += float64(a[i]) * float64(b[i])
+	}
+	return dot
+}

internal/embedding/potion.go

@@ -0,0 +1,321 @@
+package embedding
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/zzet/gortex/internal/platform"
+)
+
+// PotionProvider embeds text with a Model2Vec static embedding model —
+// a WordPiece tokenizer plus a single [vocab × dims] embedding matrix.
+// Inference is: tokenize → gather the token rows → mean-pool →
+// L2-normalise (the model config's `normalize: true` convention, matching
+// the reference implementations). No transformer, no positional limit —
+// a 50-candidate rerank batch embeds in well under a millisecond.
+//
+// The bundled model is minishlab/potion-code-16M-v2 (MIT): 256-dim
+// vectors over a ~63.5k-token code-mined vocabulary, distilled from a
+// code-retrieval teacher. It replaces the older averaged word-vector
+// channel for the rerank's semantic-cosine signal.
+type PotionProvider struct {
+	tok  *wordPieceTokenizer
+	mat  []float32 // row-major [vocab][dims]
+	dims int
+}
+
+// Potion model pin. The revision and checksums identify the exact
+// artifacts the loader accepts; a mismatched download is discarded.
+const (
+	potionModelName = "potion-code-16M-v2"
+	potionRevision  = "d3daf3e31f36d78f75913030b8bdf4a505d5b833"
+	potionBaseURL   = "https://huggingface.co/minishlab/" + potionModelName + "/resolve/" + potionRevision
+
+	potionWeightsFile   = "model.safetensors"
+	potionTokenizerFile = "tokenizer.json"
+
+	potionWeightsSHA256   = "75cf7a6c2171b230ad19b1e7d8e0b1aee86da5a02af8e7cacedd9921d227623c"
+	potionTokenizerSHA256 = "107bbdcbad4bff1d299b7a4c3a2fb17c52890688b7dd0e4c9deab79d3c4f3d45"
+)
+
+// maxPotionTokens caps how many tokens of one text feed the mean-pool.
+// The model is static (no sequence limit), but the rerank only ever
+// embeds short name+signature+doc fragments; the cap bounds the cost of
+// a pathological input.
+const maxPotionTokens = 512
+
+// NewPotionProviderFromDir loads the model from a directory holding
+// model.safetensors and tokenizer.json.
+func NewPotionProviderFromDir(dir string) (*PotionProvider, error) {
+	tok, err := loadWordPieceTokenizer(filepath.Join(dir, potionTokenizerFile))
+	if err != nil {
+		return nil, err
+	}
+	mat, dims, err := loadSafetensorsF16(filepath.Join(dir, potionWeightsFile))
+	if err != nil {
+		return nil, err
+	}
+	return &PotionProvider{tok: tok, mat: mat, dims: dims}, nil
+}
+
+func (p *PotionProvider) Dimensions() int { return p.dims }
+func (p *PotionProvider) Close() error    { return nil }
+
+func (p *PotionProvider) Embed(_ context.Context, text string) ([]float32, error) {
+	return p.embed(text), nil
+}
+
+func (p *PotionProvider) EmbedBatch(_ context.Context, texts []string) ([][]float32, error) {
+	out := make([][]float32, len(texts))
+	for i, t := range texts {
+		out[i] = p.embed(t)
+	}
+	return out, nil
+}
+
+func (p *PotionProvider) embed(text string) []float32 {
+	ids := p.tok.Encode(text)
+	if len(ids) > maxPotionTokens {
+		ids = ids[:maxPotionTokens]
+	}
+	vec := make([]float32, p.dims)
+	if len(ids) == 0 {
+		return vec
+	}
+	rows := len(p.mat) / p.dims
+	n := 0
+	for _, id := range ids {
+		if id < 0 || id >= rows {
+			continue
+		}
+		row := p.mat[id*p.dims : (id+1)*p.dims]
+		for i, v := range row {
+			vec[i] += v
+		}
+		n++
+	}
+	if n == 0 {
+		return vec
+	}
+	inv := 1.0 / float32(n)
+	var norm float64
+	for i := range vec {
+		vec[i] *= inv
+		norm += float64(vec[i]) * float64(vec[i])
+	}
+	// L2-normalise (model config `normalize: true`).
+	if norm > 0 {
+		s := float32(1.0 / math.Sqrt(norm))
+		for i := range vec {
+			vec[i] *= s
+		}
+	}
+	return vec
+}
+
+// loadSafetensorsF16 reads a single-tensor safetensors file holding an
+// F16 "embeddings" matrix and returns it as row-major float32.
+func loadSafetensorsF16(path string) ([]float32, int, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, 0, fmt.Errorf("read weights: %w", err)
+	}
+	if len(raw) < 8 {
+		return nil, 0, fmt.Errorf("weights file too short")
+	}
+	hlen := binary.LittleEndian.Uint64(raw[:8])
+	if hlen == 0 || 8+hlen > uint64(len(raw)) {
+		return nil, 0, fmt.Errorf("weights header out of range")
+	}
+	var header map[string]json.RawMessage
+	if err := json.Unmarshal(raw[8:8+hlen], &header); err != nil {
+		return nil, 0, fmt.Errorf("parse weights header: %w", err)
+	}
+	var info struct {
+		Dtype       string `json:"dtype"`
+		Shape       []int  `json:"shape"`
+		DataOffsets []int  `json:"data_offsets"`
+	}
+	entry, ok := header["embeddings"]
+	if !ok {
+		return nil, 0, fmt.Errorf("weights file has no 'embeddings' tensor")
+	}
+	if err := json.Unmarshal(entry, &info); err != nil {
+		return nil, 0, fmt.Errorf("parse tensor info: %w", err)
+	}
+	if info.Dtype != "F16" {
+		return nil, 0, fmt.Errorf("unsupported tensor dtype %q (want F16)", info.Dtype)
+	}
+	if len(info.Shape) != 2 || len(info.DataOffsets) != 2 {
+		return nil, 0, fmt.Errorf("unexpected tensor shape/offsets")
+	}
+	rows, dims := info.Shape[0], info.Shape[1]
+	start := 8 + int(hlen) + info.DataOffsets[0]
+	end := 8 + int(hlen) + info.DataOffsets[1]
+	if start < 0 || end > len(raw) || end-start != rows*dims*2 {
+		return nil, 0, fmt.Errorf("tensor data out of range")
+	}
+	data := raw[start:end]
+	mat := make([]float32, rows*dims)
+	for i := range mat {
+		mat[i] = f16ToF32(binary.LittleEndian.Uint16(data[i*2 : i*2+2]))
+	}
+	return mat, dims, nil
+}
+
+// f16ToF32 converts an IEEE-754 half-precision value to float32.
+func f16ToF32(h uint16) float32 {
+	sign := uint32(h>>15) << 31
+	exp := uint32(h>>10) & 0x1f
+	man := uint32(h) & 0x3ff
+	switch exp {
+	case 0:
+		if man == 0 {
+			return math.Float32frombits(sign)
+		}
+		// Subnormal half: renormalise into a normal float32.
+		e := uint32(113) // 127 - 14
+		for man&0x400 == 0 {
+			man <<= 1
+			e--
+		}
+		man &= 0x3ff
+		return math.Float32frombits(sign | e<<23 | man<<13)
+	case 0x1f:
+		return math.Float32frombits(sign | 0xff<<23 | man<<13)
+	default:
+		return math.Float32frombits(sign | (exp+112)<<23 | man<<13)
+	}
+}
+
+// resolvePotionDir returns the first directory that holds both model
+// files, probing in order:
+//
+//  1. $GORTEX_POTION_DIR — explicit override.
+//  2. <executable dir>/models/<model> — the release/bench sidecar. A
+//     packaged install ships the model next to the binary, so it is
+//     fully offline after install.
+//  3. <gortex home>/models/<model> — the per-user cache the first-use
+//     download fills.
+//
+// Returns "" when none has the files.
+func resolvePotionDir() string {
+	var candidates []string
+	if d := os.Getenv("GORTEX_POTION_DIR"); d != "" {
+		candidates = append(candidates, d)
+	}
+	if exe, err := os.Executable(); err == nil {
+		candidates = append(candidates, filepath.Join(filepath.Dir(exe), "models", potionModelName))
+	}
+	candidates = append(candidates, filepath.Join(platform.ModelsDir(), potionModelName))
+	for _, dir := range candidates {
+		if hasPotionFiles(dir) {
+			return dir
+		}
+	}
+	return ""
+}
+
+func hasPotionFiles(dir string) bool {
+	for _, f := range []string{potionWeightsFile, potionTokenizerFile} {
+		if st, err := os.Stat(filepath.Join(dir, f)); err != nil || st.IsDir() {
+			return false
+		}
+	}
+	return true
+}
+
+// downloadPotion fetches the pinned model revision into the per-user
+// models dir, verifying each file's SHA-256 before moving it into
+// place. Returns the directory on success. Disabled entirely when
+// GORTEX_POTION_DOWNLOAD=0 (offline installs ship the sidecar instead).
+func downloadPotion() (string, error) {
+	if v := strings.TrimSpace(os.Getenv("GORTEX_POTION_DOWNLOAD")); v == "0" || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") {
+		return "", fmt.Errorf("model download disabled by GORTEX_POTION_DOWNLOAD")
+	}
+	// Test binaries never reach the network: a `go test` run that
+	// exercises a search path must stay hermetic and fall back to the
+	// baked vectors instead of pulling 32MB mid-suite.
+	if strings.HasSuffix(os.Args[0], ".test") {
+		return "", fmt.Errorf("model download disabled in test binaries")
+	}
+	dir := filepath.Join(platform.ModelsDir(), potionModelName)
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return "", err
+	}
+	client := &http.Client{Timeout: 120 * time.Second}
+	files := []struct{ name, sum string }{
+		{potionWeightsFile, potionWeightsSHA256},
+		{potionTokenizerFile, potionTokenizerSHA256},
+	}
+	for _, f := range files {
+		dst := filepath.Join(dir, f.name)
+		if st, err := os.Stat(dst); err == nil && !st.IsDir() {
+			if ok, _ := fileSHA256Matches(dst, f.sum); ok {
+				continue
+			}
+		}
+		if err := downloadVerified(client, potionBaseURL+"/"+f.name, dst, f.sum); err != nil {
+			return "", fmt.Errorf("fetch %s: %w", f.name, err)
+		}
+	}
+	return dir, nil
+}
+
+// downloadVerified streams url into dst.partial, verifies the SHA-256,
+// and renames into place atomically.
+func downloadVerified(client *http.Client, url, dst, wantSHA string) error {
+	resp, err := client.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("HTTP %d", resp.StatusCode)
+	}
+	tmp := dst + ".partial"
+	out, err := os.Create(tmp)
+	if err != nil {
+		return err
+	}
+	h := sha256.New()
+	_, cpErr := io.Copy(io.MultiWriter(out, h), resp.Body)
+	closeErr := out.Close()
+	if cpErr != nil {
+		os.Remove(tmp)
+		return cpErr
+	}
+	if closeErr != nil {
+		os.Remove(tmp)
+		return closeErr
+	}
+	if got := hex.EncodeToString(h.Sum(nil)); got != wantSHA {
+		os.Remove(tmp)
+		return fmt.Errorf("checksum mismatch: got %s want %s", got, wantSHA)
+	}
+	return os.Rename(tmp, dst)
+}
+
+func fileSHA256Matches(path, want string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return false, err
+	}
+	return hex.EncodeToString(h.Sum(nil)) == want, nil
+}

internal/embedding/potion_test.go

@@ -0,0 +1,214 @@
+package embedding
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"math"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func writeTestTokenizer(t *testing.T, dir string, vocab map[string]int) {
+	t.Helper()
+	tj := map[string]any{
+		"normalizer": map[string]any{
+			"type": "BertNormalizer", "clean_text": true,
+			"handle_chinese_chars": true, "strip_accents": nil, "lowercase": true,
+		},
+		"pre_tokenizer": map[string]any{"type": "BertPreTokenizer"},
+		"model": map[string]any{
+			"type": "WordPiece", "unk_token": "[UNK]",
+			"continuing_subword_prefix": "##", "max_input_chars_per_word": 100,
+			"vocab": vocab,
+		},
+	}
+	raw, err := json.Marshal(tj)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, potionTokenizerFile), raw, 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// f32ToF16 converts float32 → IEEE half for fixture building. Handles
+// the normal range the fixtures use; no rounding subtleties needed.
+func f32ToF16(f float32) uint16 {
+	bits := math.Float32bits(f)
+	sign := uint16(bits>>16) & 0x8000
+	exp := int32(bits>>23&0xff) - 127 + 15
+	man := bits >> 13 & 0x3ff
+	if exp <= 0 {
+		return sign
+	}
+	if exp >= 0x1f {
+		return sign | 0x7c00
+	}
+	return sign | uint16(exp)<<10 | uint16(man)
+}
+
+func writeTestWeights(t *testing.T, dir string, rows [][]float32) {
+	t.Helper()
+	dims := len(rows[0])
+	header := map[string]any{
+		"embeddings": map[string]any{
+			"dtype": "F16", "shape": []int{len(rows), dims},
+			"data_offsets": []int{0, len(rows) * dims * 2},
+		},
+	}
+	hj, err := json.Marshal(header)
+	if err != nil {
+		t.Fatal(err)
+	}
+	buf := make([]byte, 8, 8+len(hj)+len(rows)*dims*2)
+	binary.LittleEndian.PutUint64(buf, uint64(len(hj)))
+	buf = append(buf, hj...)
+	for _, row := range rows {
+		for _, v := range row {
+			var b [2]byte
+			binary.LittleEndian.PutUint16(b[:], f32ToF16(v))
+			buf = append(buf, b[:]...)
+		}
+	}
+	if err := os.WriteFile(filepath.Join(dir, potionWeightsFile), buf, 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestWordPiece_BertSchemeHermetic(t *testing.T) {
+	dir := t.TempDir()
+	vocab := map[string]int{
+		"[PAD]": 0, "[UNK]": 1,
+		"bind": 2, "##body": 3, "body": 4, "(": 5, ")": 6, "*": 7,
+		"client": 8, "func": 9, ".": 10, "go": 11,
+	}
+	writeTestTokenizer(t, dir, vocab)
+	tok, err := loadWordPieceTokenizer(filepath.Join(dir, potionTokenizerFile))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cases := []struct {
+		text string
+		want []int
+	}{
+		// camel-cased word lowercased then greedily split
+		{"BindBody", []int{2, 3}},
+		// punctuation isolated; unknown word → UNK
+		{"func (c *Client)", []int{9, 5, 1, 7, 8, 6}},
+		// dot split as punctuation
+		{"bind.go", []int{2, 10, 11}},
+		// whitespace collapse + empty
+		{"   ", nil},
+		// no matching piece anywhere → whole-word UNK
+		{"zzzqqq", []int{1}},
+	}
+	for _, c := range cases {
+		got := tok.Encode(c.text)
+		if len(got) != len(c.want) {
+			t.Fatalf("Encode(%q) = %v, want %v", c.text, got, c.want)
+		}
+		for i := range got {
+			if got[i] != c.want[i] {
+				t.Fatalf("Encode(%q) = %v, want %v", c.text, got, c.want)
+			}
+		}
+	}
+}
+
+func TestPotionProvider_MeanPoolAndNormalize(t *testing.T) {
+	dir := t.TempDir()
+	vocab := map[string]int{"[PAD]": 0, "[UNK]": 1, "bind": 2, "##body": 3}
+	writeTestTokenizer(t, dir, vocab)
+	writeTestWeights(t, dir, [][]float32{
+		{0, 0, 0, 0},  // PAD
+		{1, 0, 0, 0},  // UNK
+		{0, 2, 0, 0},  // bind
+		{0, 0, 2, 0},  // ##body
+	})
+	p, err := NewPotionProviderFromDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p.Dimensions() != 4 {
+		t.Fatalf("dims = %d, want 4", p.Dimensions())
+	}
+	vec, err := p.Embed(context.Background(), "BindBody")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// mean of (0,2,0,0) and (0,0,2,0) = (0,1,1,0); L2-normalised.
+	want := []float32{0, float32(1 / math.Sqrt2), float32(1 / math.Sqrt2), 0}
+	for i := range want {
+		if math.Abs(float64(vec[i]-want[i])) > 1e-3 {
+			t.Fatalf("vec = %v, want %v", vec, want)
+		}
+	}
+	// Unknown-only text embeds the UNK row, not a zero vector.
+	vec2, _ := p.Embed(context.Background(), "zzzqqq")
+	if vec2[0] < 0.99 {
+		t.Fatalf("UNK-only embed = %v, want unit x-axis", vec2)
+	}
+	// Empty text → zero vector (signal reads it as no evidence).
+	vec3, _ := p.Embed(context.Background(), "")
+	for _, v := range vec3 {
+		if v != 0 {
+			t.Fatalf("empty embed should be zero vector, got %v", vec3)
+		}
+	}
+}
+
+// TestPotionProvider_GoldenAgainstReference verifies the pure-Go
+// tokenizer + pooling against reference outputs captured from the
+// upstream tokenizers + numpy implementation for the real model.
+// Skipped when the model files are not installed on this machine.
+func TestPotionProvider_GoldenAgainstReference(t *testing.T) {
+	dir := resolvePotionDir()
+	if dir == "" {
+		t.Skip("potion model files not installed; run with GORTEX_POTION_DIR pointing at the model")
+	}
+	p, err := NewPotionProviderFromDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	golden := []struct {
+		text  string
+		ids   []int
+		head8 []float32
+	}{
+		{
+			"BindBody bsonBinding.BindBody binding/bson.go",
+			[]int{13190, 22687, 30716, 7431, 3670, 15, 13190, 22687, 7034, 16, 30716, 15, 1178},
+			[]float32{0.034474, 0.018522, -0.13514, 0.084147, 0.232714, -0.36536, 0.193378, 0.139676},
+		},
+		{
+			"decode bson request body",
+			[]int{29631, 30716, 4230, 1306},
+			[]float32{-0.07553, -0.022155, -0.119157, 0.177616, 0.124399, -0.224746, 0.050767, 0.065148},
+		},
+		{
+			"func (c *Client) Do(req *Request) (*Response, error)",
+			[]int{29527, 9, 42, 11, 6399, 10, 1082, 9, 29556, 11, 4230, 10, 9, 11, 2436, 13, 6564, 10},
+			[]float32{-0.087694, 0.013899, -0.003767, 0.109125, -0.006322, 0.076795, 0.031422, 0.040902},
+		},
+	}
+	for _, g := range golden {
+		ids := p.tok.Encode(g.text)
+		if len(ids) != len(g.ids) {
+			t.Fatalf("Encode(%q) ids = %v, want %v", g.text, ids, g.ids)
+		}
+		for i := range ids {
+			if ids[i] != g.ids[i] {
+				t.Fatalf("Encode(%q) ids = %v, want %v", g.text, ids, g.ids)
+			}
+		}
+		vec, _ := p.Embed(context.Background(), g.text)
+		for i, w := range g.head8 {
+			if math.Abs(float64(vec[i]-w)) > 2e-3 {
+				t.Fatalf("Embed(%q)[%d] = %.6f, want %.6f", g.text, i, vec[i], w)
+			}
+		}
+	}
+}

internal/embedding/provider.go

@@ -0,0 +1,203 @@
+// Package embedding provides pluggable embedding providers for semantic search.
+//
+// The default build includes the Hugot provider (pure-Go ONNX runtime via
+// hugot.NewGoSession) which auto-downloads MiniLM-L6-v2 on first use — no
+// external runtime, no manual model placement. The legacy StaticProvider
+// (GloVe word vectors) and APIProvider (Ollama/OpenAI) are also always
+// available.
+//
+// Opt-in build tags enable faster transformer backends for users who are
+// willing to manage native dependencies:
+//   - embeddings_onnx  — yalue/onnxruntime_go with libonnxruntime on PATH
+//   - embeddings_gomlx — hugot with XLA/PJRT plugin (~100MB auto-download)
+package embedding
+
+import (
+	"context"
+	"errors"
+	"fmt"
+)
+
+// ErrBackendNotCompiled marks a local-backend factory that failed only because
+// its build tag was not set (the onnx/gomlx stubs). Such a failure is benign
+// noise in a default build — callers use errors.Is to log it at debug rather
+// than warn even when the chain degrades to the static fallback.
+var ErrBackendNotCompiled = errors.New("embedding backend not compiled in")
+
+// Provider generates embedding vectors from text.
+type Provider interface {
+	// Embed returns the embedding vector for the given text.
+	Embed(ctx context.Context, text string) ([]float32, error)
+
+	// EmbedBatch returns embeddings for multiple texts.
+	EmbedBatch(ctx context.Context, texts []string) ([][]float32, error)
+
+	// Dimensions returns the embedding vector size.
+	Dimensions() int
+
+	// Close releases resources.
+	Close() error
+}
+
+// NewHugotProvider exposes the pure-Go Hugot backend (MiniLM-L6-v2)
+// directly, without the NewLocalProvider fallback chain. Useful when a
+// caller wants a hard error if Hugot can't start (e.g. eval harnesses
+// that mustn't silently degrade to static GloVe).
+func NewHugotProvider() (Provider, error) { return newHugotProvider() }
+
+// NewHugotProviderWithVariant loads a specific embedder variant from
+// any registered HuggingFace repo (MiniLM variants, code-tuned models,
+// …). Pass a name returned by KnownHugotVariants (e.g. "fp32",
+// "qint8_arm64", "jina_code", "bge_code"). Returns an error if the
+// variant name is unknown or the download/load fails.
+func NewHugotProviderWithVariant(variant string) (Provider, error) {
+	v, ok := LookupHugotVariant(variant)
+	if !ok {
+		return nil, fmt.Errorf("unknown hugot variant %q (known: %v)", variant, KnownHugotVariants())
+	}
+	return newHugotProviderWithSpec(v)
+}
+
+// ProviderConfig is the subset of an embedding configuration that
+// NewProviderFromConfig needs. It is a local struct — not the
+// config.EmbeddingConfig type — so the embedding package stays free of
+// an import dependency on internal/config. Callers translate their
+// config block into this shape.
+type ProviderConfig struct {
+	// Provider selects the backend: "static" (baked GloVe, the
+	// default), "local" (best available transformer), or "api" (an
+	// external embedding endpoint). Empty is treated as "static".
+	Provider string
+	// APIURL / APIModel parameterise the "api" provider.
+	APIURL   string
+	APIModel string
+	// Variant names a specific local transformer model to load (a key
+	// from KnownHugotVariants, e.g. "fp32", "bge_small", "jina_code").
+	// Honoured only when Provider is "local": a non-empty Variant pins
+	// that exact model via NewHugotProviderWithVariant instead of the
+	// auto-selected NewLocalProvider backend. Empty preserves the
+	// existing default-selection behaviour. Ignored for other providers.
+	Variant string
+}
+
+// NewProviderFromConfig constructs an embedding provider from a
+// configuration block. The selection logic:
+//
+//   - "static" (or empty)  → NewStaticProvider — baked GloVe word
+//     vectors, zero download, CPU-only. This is the default because
+//     it makes semantic search work with no setup.
+//   - "local"              → NewLocalProvider — the best available
+//     transformer backend (Hugot MiniLM auto-downloads on first use).
+//     When cfg.Variant names a specific model, that exact variant is
+//     loaded via NewHugotProviderWithVariant instead.
+//   - "api"                → NewAPIProvider against cfg.APIURL.
+//
+// An unknown provider name is an error so a typo in `.gortex.yaml`
+// fails loudly instead of silently degrading.
+func NewProviderFromConfig(cfg ProviderConfig) (Provider, error) {
+	switch cfg.Provider {
+	case "", "static":
+		return NewStaticProvider()
+	case "local":
+		// A pinned variant loads that exact model; an empty variant
+		// keeps the existing auto-selection (ONNX → GoMLX → Hugot →
+		// static) so every prior config behaves identically.
+		if cfg.Variant != "" {
+			return NewHugotProviderWithVariant(cfg.Variant)
+		}
+		return NewLocalProvider()
+	case "api":
+		if cfg.APIURL == "" {
+			return nil, fmt.Errorf("embedding provider %q requires an api_url", cfg.Provider)
+		}
+		return NewAPIProvider(cfg.APIURL, cfg.APIModel), nil
+	default:
+		return nil, fmt.Errorf("unknown embedding provider %q (want static, local, or api)", cfg.Provider)
+	}
+}
+
+// NewProviderFromConfigWithReport is NewProviderFromConfig plus a SelectionReport
+// for the auto-selected local backend (Provider "local" with no pinned Variant),
+// so the caller can log which backend was constructed and which were skipped.
+// For every other provider the report is empty.
+func NewProviderFromConfigWithReport(cfg ProviderConfig) (Provider, SelectionReport, error) {
+	if cfg.Provider == "local" && cfg.Variant == "" {
+		p, report := NewLocalProviderWithReport()
+		if p == nil {
+			err := fmt.Errorf("no embedding provider available")
+			if n := len(report.Attempts); n > 0 {
+				err = report.Attempts[n-1].Err
+			}
+			return nil, report, err
+		}
+		return p, report, nil
+	}
+	p, err := NewProviderFromConfig(cfg)
+	return p, SelectionReport{}, err
+}
+
+// SelectionAttempt records one backend the local-provider chain tried and the
+// error that made it fall through. It exists so a silent degradation to the
+// static GloVe fallback becomes observable to the caller.
+type SelectionAttempt struct {
+	Backend string
+	Err     error
+}
+
+// SelectionReport describes how NewLocalProviderWithReport chose a backend: the
+// backend actually constructed, its dimension, and every rejected attempt.
+// Chosen is the backend name (e.g. "hugot", "static"); it is "static" when the
+// chain fell all the way through to the GloVe fallback.
+type SelectionReport struct {
+	Chosen   string
+	Dims     int
+	Attempts []SelectionAttempt
+}
+
+// NewLocalProviderWithReport returns the best available local embedding provider
+// along with a report of every backend it tried. Preference order: ONNX
+// (fastest, requires libonnxruntime) → GoMLX (XLA) → Hugot (pure Go, always
+// compiled in) → Static (GloVe word vectors fallback). A nil provider means even
+// the static fallback failed to construct (its error is the last attempt).
+func NewLocalProviderWithReport() (Provider, SelectionReport) {
+	factories := []struct {
+		name    string
+		factory func() (Provider, error)
+	}{
+		{"onnx", newONNXProvider},
+		{"gomlx", newGoMLXProvider},
+		{"hugot", newHugotProvider},
+	}
+	var report SelectionReport
+	for _, nf := range factories {
+		p, err := nf.factory()
+		if err == nil {
+			report.Chosen = nf.name
+			report.Dims = p.Dimensions()
+			return p, report
+		}
+		report.Attempts = append(report.Attempts, SelectionAttempt{Backend: nf.name, Err: err})
+	}
+	// Fallback: static word vectors (always available, no network).
+	p, err := NewStaticProvider()
+	if err != nil {
+		report.Attempts = append(report.Attempts, SelectionAttempt{Backend: "static", Err: err})
+		return nil, report
+	}
+	report.Chosen = "static"
+	report.Dims = p.Dimensions()
+	return p, report
+}
+
+// NewLocalProvider returns the best available local embedding provider,
+// discarding the selection report. See NewLocalProviderWithReport.
+func NewLocalProvider() (Provider, error) {
+	p, report := NewLocalProviderWithReport()
+	if p == nil {
+		if n := len(report.Attempts); n > 0 {
+			return nil, report.Attempts[n-1].Err
+		}
+		return nil, fmt.Errorf("no embedding provider available")
+	}
+	return p, nil
+}

internal/embedding/provider_config_test.go

@@ -0,0 +1,61 @@
+package embedding
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestNewProviderFromConfig_DefaultIsStatic asserts that an empty
+// provider name — the zero value, which is what an unconfigured
+// `embedding:` block yields — selects the static GloVe provider. This
+// is the default-on path: semantic search works with no setup.
+func TestNewProviderFromConfig_DefaultIsStatic(t *testing.T) {
+	p, err := NewProviderFromConfig(ProviderConfig{})
+	require.NoError(t, err)
+	defer p.Close()
+
+	_, isStatic := p.(*StaticProvider)
+	assert.True(t, isStatic, "empty provider name must select the static GloVe provider")
+	assert.Equal(t, 50, p.Dimensions(), "static GloVe is 50-dimensional")
+}
+
+// TestNewProviderFromConfig_ExplicitStatic asserts that the explicit
+// "static" name also selects the static provider.
+func TestNewProviderFromConfig_ExplicitStatic(t *testing.T) {
+	p, err := NewProviderFromConfig(ProviderConfig{Provider: "static"})
+	require.NoError(t, err)
+	defer p.Close()
+
+	_, isStatic := p.(*StaticProvider)
+	assert.True(t, isStatic)
+}
+
+// TestNewProviderFromConfig_API asserts that the "api" provider
+// constructs an APIProvider against the configured URL, and that a
+// missing URL is a hard error rather than a silent fallback.
+func TestNewProviderFromConfig_API(t *testing.T) {
+	p, err := NewProviderFromConfig(ProviderConfig{
+		Provider: "api",
+		APIURL:   "http://localhost:11434",
+		APIModel: "nomic-embed-text",
+	})
+	require.NoError(t, err)
+	defer p.Close()
+
+	api, isAPI := p.(*APIProvider)
+	require.True(t, isAPI, "the api provider must construct an APIProvider")
+	assert.Equal(t, "nomic-embed-text", api.model)
+
+	_, err = NewProviderFromConfig(ProviderConfig{Provider: "api"})
+	require.Error(t, err, "the api provider without a URL must be an error")
+}
+
+// TestNewProviderFromConfig_UnknownProviderErrors asserts that a typo
+// in the provider name fails loudly instead of degrading silently.
+func TestNewProviderFromConfig_UnknownProviderErrors(t *testing.T) {
+	_, err := NewProviderFromConfig(ProviderConfig{Provider: "transfromer"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "transfromer")
+}

internal/embedding/provider_test.go

@@ -0,0 +1,164 @@
+package embedding
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStaticProvider_Embed(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+
+	// Inject test vectors.
+	p.SetVectors(map[string][]float32{
+		"validate": {1, 0, 0},
+		"check":    {0.9, 0.1, 0},
+		"token":    {0, 1, 0},
+		"auth":     {0, 0.9, 0.1},
+	})
+
+	ctx := context.Background()
+
+	// "validate token" should produce a non-zero vector.
+	vec, err := p.Embed(ctx, "validateToken")
+	require.NoError(t, err)
+	assert.Len(t, vec, 3)
+	assert.NotEqual(t, float32(0), vec[0], "should have non-zero components")
+
+	// "check auth" should be similar to "validate token".
+	vec2, err := p.Embed(ctx, "checkAuth")
+	require.NoError(t, err)
+	assert.Len(t, vec2, 3)
+
+	// Both should be non-zero (found words in vocabulary).
+	hasNonZero := false
+	for _, v := range vec2 {
+		if v != 0 {
+			hasNonZero = true
+			break
+		}
+	}
+	assert.True(t, hasNonZero, "should find words in vocabulary")
+}
+
+func TestStaticProvider_EmbedBatch(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	p.SetVectors(map[string][]float32{
+		"hello": {1, 0},
+		"world": {0, 1},
+	})
+
+	ctx := context.Background()
+	vecs, err := p.EmbedBatch(ctx, []string{"hello", "world", "unknown"})
+	require.NoError(t, err)
+	assert.Len(t, vecs, 3)
+}
+
+func TestStaticProvider_UnknownWords(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	p.SetVectors(map[string][]float32{
+		"known": {1, 0},
+	})
+
+	ctx := context.Background()
+	vec, err := p.Embed(ctx, "completelyunknownword")
+	require.NoError(t, err)
+	// Should return zero vector for unknown words.
+	for _, v := range vec {
+		assert.Equal(t, float32(0), v)
+	}
+}
+
+func TestNopProvider(t *testing.T) {
+	var p NopProvider
+	ctx := context.Background()
+
+	_, err := p.Embed(ctx, "test")
+	assert.ErrorIs(t, err, ErrDisabled)
+
+	_, err = p.EmbedBatch(ctx, []string{"test"})
+	assert.ErrorIs(t, err, ErrDisabled)
+
+	assert.Equal(t, 0, p.Dimensions())
+	assert.NoError(t, p.Close())
+}
+
+func TestStaticProvider_SemanticSimilarity(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	require.Greater(t, len(p.vectors), 1000, "should have loaded GloVe vectors")
+
+	ctx := context.Background()
+
+	// "validate token" and "check authentication" should produce non-zero, similar vectors.
+	vec1, err := p.Embed(ctx, "validate token")
+	require.NoError(t, err)
+
+	vec2, err := p.Embed(ctx, "check authentication")
+	require.NoError(t, err)
+
+	// Both should be non-zero.
+	nonZero1 := false
+	for _, v := range vec1 {
+		if v != 0 {
+			nonZero1 = true
+			break
+		}
+	}
+	assert.True(t, nonZero1, "validate token should produce non-zero vector")
+
+	// Compute cosine similarity.
+	var dot float64
+	for i := range vec1 {
+		dot += float64(vec1[i]) * float64(vec2[i])
+	}
+	// Both vectors are normalized, so dot product = cosine similarity.
+	assert.Greater(t, dot, 0.3, "semantically similar queries should have cosine > 0.3")
+}
+
+func TestNewLocalProvider_ReturnsWorkingProvider(t *testing.T) {
+	if raceEnabled {
+		// NewLocalProvider falls through to Hugot on a fresh machine,
+		// which triggers hugot.DownloadModel → go-huggingface's
+		// DownloadFilesCtx. That function has a data race on a shared
+		// loop variable in its internal worker goroutines (upstream
+		// bug; reproduces with every `hub.Repo.DownloadFiles` call).
+		// We don't own that code and the race doesn't affect our own
+		// logic, so skip under -race rather than bundle a vendored
+		// patch. Non-race builds still exercise the full fallback.
+		t.Skip("upstream data race in go-huggingface/hub.DownloadFilesCtx — skipping under -race")
+	}
+	p, err := NewLocalProvider()
+	require.NoError(t, err)
+	defer p.Close()
+
+	// Default build walks ONNX → GoMLX → Hugot → Static and returns
+	// the first that initialises. With Hugot's onnx file pinned to
+	// onnx/model.onnx, Hugot succeeds when the model is cached or
+	// the network is reachable. Either is fine — the invariant is
+	// "NewLocalProvider returns a working provider."
+	assert.NotNil(t, p)
+	assert.Greater(t, p.Dimensions(), 0, "provider must report positive dimensions")
+}
+
+func TestTokenizeForEmbedding(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected []string
+	}{
+		{"validateToken", []string{"validate", "token"}},
+		{"get_user_by_id", []string{"get", "user", "by", "id"}},
+		{"internal/auth/service.go", []string{"internal", "auth", "service", "go"}},
+		{"a b", nil}, // single chars dropped
+		{"HandleRequest", []string{"handle", "request"}},
+	}
+	for _, tt := range tests {
+		got := tokenizeForEmbedding(tt.input)
+		assert.Equal(t, tt.expected, got, "tokenize(%q)", tt.input)
+	}
+}

internal/embedding/race_off.go

@@ -0,0 +1,5 @@
+//go:build !race
+
+package embedding
+
+const raceEnabled = false

internal/embedding/race_on.go

@@ -0,0 +1,9 @@
+//go:build race
+
+package embedding
+
+// raceEnabled is true when the binary was built with the Go race
+// detector. Used to skip tests whose failure mode is an upstream
+// library race we don't control (go-huggingface/hub.DownloadFilesCtx),
+// rather than a real bug in our own code.
+const raceEnabled = true

internal/embedding/shared.go

@@ -0,0 +1,96 @@
+package embedding
+
+import (
+	"context"
+	"os"
+	"strings"
+	"sync"
+)
+
+// sharedStatic memoises a single process-wide StaticProvider. The baked
+// GloVe vectors are ~3.7MB compressed and decompress into a ~20k-entry
+// map that is safe for concurrent reads, so one instance serves every
+// rerank call. Constructed lazily on first use.
+var (
+	sharedStaticOnce sync.Once
+	sharedStaticInst *StaticProvider
+)
+
+// SharedStatic returns the process-wide static word-vector provider,
+// constructing it on first call. Returns nil only when the baked
+// vectors fail to load (a corrupt build); callers treat nil as "no
+// semantic-cosine channel". Safe for concurrent use.
+func SharedStatic() *StaticProvider {
+	sharedStaticOnce.Do(func() {
+		p, err := NewStaticProvider()
+		if err != nil {
+			return
+		}
+		sharedStaticInst = p
+	})
+	return sharedStaticInst
+}
+
+// EmbedTextFunc adapts a provider into the plain func the rerank
+// Context wants: text -> normalised vector, errors and nil providers
+// collapsing to a nil result the signal reads as "cannot embed".
+func EmbedTextFunc(p Provider) func(string) []float32 {
+	if p == nil {
+		return nil
+	}
+	return func(text string) []float32 {
+		vec, err := p.Embed(context.Background(), text)
+		if err != nil {
+			return nil
+		}
+		return vec
+	}
+}
+
+// sharedCode memoises the process-wide code-embedding provider used by
+// the rerank's semantic-cosine channel: the bundled static code model
+// (potion) when its files resolve — explicit dir, exec-adjacent
+// sidecar, per-user models dir, or a checksum-verified first-use
+// download — and the baked GloVe word vectors otherwise, so an offline
+// install without the sidecar still gets a semantic channel.
+var (
+	sharedCodeOnce sync.Once
+	sharedCodeInst Provider
+)
+
+// SharedCodeEmbedder returns the process-wide code embedder for the
+// rerank's semantic-cosine channel. Never returns an error — the
+// fallback chain ends at the baked static provider; nil only when even
+// that failed to load. Safe for concurrent use. GORTEX_POTION=0 pins
+// the GloVe fallback (diagnostic escape hatch).
+func SharedCodeEmbedder() Provider {
+	sharedCodeOnce.Do(func() {
+		if v := strings.TrimSpace(os.Getenv("GORTEX_POTION")); v == "0" || strings.EqualFold(v, "false") || strings.EqualFold(v, "off") {
+			sharedCodeInst = staticOrNil()
+			return
+		}
+		dir := resolvePotionDir()
+		if dir == "" {
+			if d, err := downloadPotion(); err == nil {
+				dir = d
+			}
+		}
+		if dir != "" {
+			if p, err := NewPotionProviderFromDir(dir); err == nil {
+				sharedCodeInst = p
+				return
+			}
+		}
+		sharedCodeInst = staticOrNil()
+	})
+	return sharedCodeInst
+}
+
+// staticOrNil adapts SharedStatic's concrete return into the Provider
+// interface without wrapping a typed nil.
+func staticOrNil() Provider {
+	if p := SharedStatic(); p != nil {
+		return p
+	}
+	return nil
+}

internal/embedding/static.go

@@ -0,0 +1,199 @@
+package embedding
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"math"
+	"strings"
+	"sync"
+)
+
+// StaticProvider computes embeddings by averaging pre-trained word vectors.
+// It provides basic semantic search — understands "validate" ≈ "check" but
+// has no contextual understanding. Always available, zero external dependencies.
+type StaticProvider struct {
+	vectors map[string][]float32
+	dims    int
+	mu      sync.RWMutex
+}
+
+// NewStaticProvider creates a provider using built-in word vectors.
+func NewStaticProvider() (*StaticProvider, error) {
+	p := &StaticProvider{
+		vectors: make(map[string][]float32),
+		dims:    50, // default GloVe 50d; overridden by loadVectors
+	}
+	if err := p.loadVectors(); err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+func (p *StaticProvider) Embed(_ context.Context, text string) ([]float32, error) {
+	tokens := tokenizeForEmbedding(text)
+	return p.averageVectors(tokens), nil
+}
+
+func (p *StaticProvider) EmbedBatch(_ context.Context, texts []string) ([][]float32, error) {
+	results := make([][]float32, len(texts))
+	for i, text := range texts {
+		tokens := tokenizeForEmbedding(text)
+		results[i] = p.averageVectors(tokens)
+	}
+	return results, nil
+}
+
+func (p *StaticProvider) Dimensions() int { return p.dims }
+func (p *StaticProvider) Close() error    { return nil }
+
+func (p *StaticProvider) averageVectors(tokens []string) []float32 {
+	result := make([]float32, p.dims)
+	count := 0
+
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+
+	for _, tok := range tokens {
+		vec, ok := p.vectors[tok]
+		if !ok {
+			continue
+		}
+		for i, v := range vec {
+			result[i] += v
+		}
+		count++
+	}
+
+	if count == 0 {
+		return result // zero vector
+	}
+
+	// Average and normalize.
+	for i := range result {
+		result[i] /= float32(count)
+	}
+	return normalize(result)
+}
+
+func normalize(v []float32) []float32 {
+	var norm float64
+	for _, x := range v {
+		norm += float64(x) * float64(x)
+	}
+	norm = math.Sqrt(norm)
+	if norm < 1e-10 {
+		return v
+	}
+	for i := range v {
+		v[i] /= float32(norm)
+	}
+	return v
+}
+
+// tokenizeForEmbedding splits text into lowercase tokens suitable for
+// word vector lookup. Splits on camelCase, underscores, dots, slashes.
+func tokenizeForEmbedding(text string) []string {
+	var tokens []string
+	var current strings.Builder
+
+	flush := func() {
+		if current.Len() >= 2 {
+			tokens = append(tokens, current.String())
+		}
+		current.Reset()
+	}
+
+	prev := rune(0)
+	for _, r := range text {
+		switch {
+		case r >= 'A' && r <= 'Z':
+			// camelCase boundary: flush before uppercase
+			if prev >= 'a' && prev <= 'z' {
+				flush()
+			}
+			current.WriteRune(r + 32) // toLower
+		case r >= 'a' && r <= 'z':
+			current.WriteRune(r)
+		case r >= '0' && r <= '9':
+			current.WriteRune(r)
+		default:
+			flush()
+		}
+		prev = r
+	}
+	flush()
+
+	return tokens
+}
+
+// loadVectors loads GloVe word vectors from the embedded data file.
+func (p *StaticProvider) loadVectors() error {
+	if len(vectorData) == 0 {
+		return nil // no embedded vectors, empty vocabulary
+	}
+
+	gz, err := gzip.NewReader(bytes.NewReader(vectorData))
+	if err != nil {
+		return fmt.Errorf("decompress vectors: %w", err)
+	}
+	defer gz.Close()
+
+	data, err := io.ReadAll(gz)
+	if err != nil {
+		return fmt.Errorf("read vectors: %w", err)
+	}
+
+	if len(data) < 8 {
+		return fmt.Errorf("vector data too short")
+	}
+
+	wordCount := binary.LittleEndian.Uint32(data[0:4])
+	dims := binary.LittleEndian.Uint32(data[4:8])
+	p.dims = int(dims)
+
+	offset := 8
+	for i := uint32(0); i < wordCount; i++ {
+		if offset+2 > len(data) {
+			break
+		}
+		wordLen := int(binary.LittleEndian.Uint16(data[offset : offset+2]))
+		offset += 2
+
+		if offset+wordLen > len(data) {
+			break
+		}
+		word := string(data[offset : offset+wordLen])
+		offset += wordLen
+
+		vecBytes := int(dims) * 4
+		if offset+vecBytes > len(data) {
+			break
+		}
+		vec := make([]float32, dims)
+		for j := range vec {
+			vec[j] = math.Float32frombits(binary.LittleEndian.Uint32(data[offset+j*4 : offset+j*4+4]))
+		}
+		offset += vecBytes
+
+		p.vectors[word] = vec
+	}
+
+	return nil
+}
+
+// SetVectors allows injecting word vectors for testing.
+func (p *StaticProvider) SetVectors(vecs map[string][]float32) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.vectors = vecs
+	if len(vecs) > 0 {
+		for _, v := range vecs {
+			p.dims = len(v)
+			break
+		}
+	}
+}

internal/embedding/static_data.go

@@ -0,0 +1,9 @@
+package embedding
+
+import _ "embed"
+
+// Word vector data: GloVe 6B, 50 dimensions, top 20k words (~3.7MB compressed).
+// Format: header (word_count uint32 + dims uint32) + entries (word_len uint16 + word + dims×float32).
+
+//go:embed data/vectors.bin.gz
+var vectorData []byte

internal/embedding/static_test.go

@@ -0,0 +1,171 @@
+package embedding
+
+import (
+	"context"
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// cosine returns the cosine similarity of two equal-length vectors.
+// Both inputs from StaticProvider are L2-normalized, so this is just
+// the dot product — but the explicit norm keeps the helper correct
+// for any caller.
+func cosine(a, b []float32) float64 {
+	var dot, na, nb float64
+	for i := range a {
+		dot += float64(a[i]) * float64(b[i])
+		na += float64(a[i]) * float64(a[i])
+		nb += float64(b[i]) * float64(b[i])
+	}
+	if na == 0 || nb == 0 {
+		return 0
+	}
+	return dot / (math.Sqrt(na) * math.Sqrt(nb))
+}
+
+// TestStaticProvider_LoadsBakedTable asserts the embedded GloVe table
+// loads and reports a plausible dimensionality and vocabulary size.
+func TestStaticProvider_LoadsBakedTable(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	defer p.Close()
+
+	// The baked table is GloVe 6B 50d. Dimensions() must reflect the
+	// loaded table, not the 50 hardcoded in the constructor before
+	// loadVectors runs.
+	assert.Equal(t, 50, p.Dimensions(), "GloVe 6B baked table is 50-dimensional")
+	assert.Greater(t, len(p.vectors), 10_000, "baked table should carry the top-~20k vocabulary")
+}
+
+// TestStaticProvider_SemanticRanking asserts that semantically related
+// code tokens land closer in vector space than unrelated ones — the
+// core property that makes static embeddings a useful fusion signal.
+//
+// The word pairs are drawn from the baked GloVe top-20k vocabulary
+// (verb pairs like validate/delete fall below GloVe 6B's frequency
+// cutoff and are absent — the tokenizer would lower them to a zero
+// vector, so the test uses present near-synonyms instead).
+func TestStaticProvider_SemanticRanking(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	defer p.Close()
+	ctx := context.Background()
+
+	embed := func(s string) []float32 {
+		v, err := p.Embed(ctx, s)
+		require.NoError(t, err)
+		require.NotEmpty(t, v)
+		// A non-zero vector confirms the token was found in-vocab; a
+		// miss would average to all-zero and make the comparison moot.
+		nonZero := false
+		for _, x := range v {
+			if x != 0 {
+				nonZero = true
+				break
+			}
+		}
+		require.True(t, nonZero, "token %q must be in the baked vocabulary", s)
+		return v
+	}
+
+	cases := []struct {
+		related, synonym, unrelated string
+	}{
+		{"check", "verify", "banana"},
+		{"request", "response", "banana"},
+		{"file", "directory", "mountain"},
+	}
+	for _, c := range cases {
+		base := embed(c.related)
+		syn := embed(c.synonym)
+		unrel := embed(c.unrelated)
+		simRelated := cosine(base, syn)
+		simUnrelated := cosine(base, unrel)
+		assert.Greater(t, simRelated, simUnrelated,
+			"%q should be closer to %q (%.3f) than to %q (%.3f)",
+			c.related, c.synonym, simRelated, c.unrelated, simUnrelated)
+	}
+}
+
+// TestStaticProvider_EmbedBatchRoundTrip asserts EmbedBatch returns one
+// vector per input in order, matching the per-item Embed result.
+func TestStaticProvider_EmbedBatchRoundTrip(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	defer p.Close()
+	ctx := context.Background()
+
+	inputs := []string{"validate token", "delete user", "parse json"}
+	batch, err := p.EmbedBatch(ctx, inputs)
+	require.NoError(t, err)
+	require.Len(t, batch, len(inputs))
+
+	for i, in := range inputs {
+		want, err := p.Embed(ctx, in)
+		require.NoError(t, err)
+		require.Len(t, batch[i], p.Dimensions())
+		assert.Equal(t, want, batch[i], "EmbedBatch[%d] must match Embed(%q)", i, in)
+	}
+}
+
+// TestStaticProvider_EmbedBatchEmpty asserts the empty-input edge case
+// returns an empty (non-nil-shaped) slice without error.
+func TestStaticProvider_EmbedBatchEmpty(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	defer p.Close()
+
+	out, err := p.EmbedBatch(context.Background(), nil)
+	require.NoError(t, err)
+	assert.Empty(t, out)
+}
+
+// TestTokenizeForEmbedding_Splitters covers every boundary the
+// tokenizer must split on: camelCase, underscore, dot, slash, and
+// mixed identifiers — plus the single-char drop rule.
+func TestTokenizeForEmbedding_Splitters(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  []string
+	}{
+		{"camelCase", "validateUserToken", []string{"validate", "user", "token"}},
+		{"underscore", "get_user_by_id", []string{"get", "user", "by", "id"}},
+		{"dotPath", "config.search.weights", []string{"config", "search", "weights"}},
+		{"slashPath", "internal/embedding/static.go", []string{"internal", "embedding", "static", "go"}},
+		{"mixed", "internal/auth.checkAccessToken", []string{"internal", "auth", "check", "access", "token"}},
+		{"singleCharsDropped", "a b c d", nil},
+		{"leadingUpper", "HTTPHandler", []string{"httphandler"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tokenizeForEmbedding(tt.input)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+// TestStaticProvider_SetVectorsInjection asserts SetVectors swaps the
+// table and re-derives Dimensions from the injected vectors — the
+// deterministic-injection path used by other tests.
+func TestStaticProvider_SetVectorsInjection(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	defer p.Close()
+
+	p.SetVectors(map[string][]float32{
+		"alpha": {1, 0, 0, 0},
+		"beta":  {0, 1, 0, 0},
+	})
+	assert.Equal(t, 4, p.Dimensions(), "Dimensions must follow the injected vector width")
+
+	vec, err := p.Embed(context.Background(), "alpha")
+	require.NoError(t, err)
+	require.Len(t, vec, 4)
+	// Single in-vocab token: averaging then normalizing a unit basis
+	// vector returns it unchanged.
+	assert.InDelta(t, 1.0, vec[0], 1e-6)
+}

internal/embedding/truncate.go

@@ -0,0 +1,188 @@
+package embedding
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"unicode/utf8"
+
+	"github.com/gomlx/go-huggingface/tokenizers/api"
+	"github.com/gomlx/go-huggingface/tokenizers/hftokenizer"
+)
+
+const (
+	// fallbackTokenBudget is the token budget used when a model directory has no
+	// parseable config.json to read max_position_embeddings from. 512 - 2 = 510,
+	// matching the classic BERT/MiniLM context window minus the two special tokens.
+	fallbackTokenBudget = 510
+
+	// specialTokenReserve is the number of positions the inference pipeline consumes
+	// for special tokens ([CLS]/[SEP]) appended after client-side truncation, so the
+	// truncation budget is max_position_embeddings minus this.
+	specialTokenReserve = 2
+
+	// runeClampBudgetFactor caps the rune-clamp fallback at budget runes. A
+	// WordPiece token spans at least one rune, so token_count <= rune_count;
+	// clamping to budget runes therefore guarantees token_count <= budget. A
+	// looser cap (e.g. 4*budget) would NOT bound the token count and could let a
+	// token-dense input (CJK, single-char words) overflow the window it is meant
+	// to protect. Recall loss in this rare fallback is an acceptable price for a
+	// hard safety bound.
+	runeClampBudgetFactor = 1
+)
+
+// tokenTruncator caps input texts at a model's positional budget before they reach
+// the inference pipeline.
+//
+// The pure-Go Hugot tokenizer path does not honour max_position_embeddings, so a
+// text longer than the model's context window reaches inference at full length and
+// produces a tensor shape mismatch that aborts the entire vector index. Truncating
+// client-side keeps that failure from ever occurring; because a transformer cannot
+// attend past its positional budget, cutting the tail is lossless by construction.
+//
+// A tokenTruncator returned by newTokenTruncator is always safe to use: if the real
+// tokenizer could not be loaded, tk is nil and Truncate degrades to a rune clamp
+// rather than disabling truncation (and the caller's local backend) entirely.
+type tokenTruncator struct {
+	tk     *hftokenizer.Tokenizer // nil ⇒ rune-clamp fallback
+	budget int                    // max token count before truncation (max_position_embeddings - 2)
+	clamp  int                    // rune cap used when tk == nil
+}
+
+// newTokenTruncator builds a truncator for the model cached under modelDir. It reads
+// <modelDir>/tokenizer.json (the same file hugot loads) and derives the token budget
+// from <modelDir>/config.json's max_position_embeddings.
+//
+// The returned truncator is always non-nil and usable. A non-nil error is
+// informational: it reports a degraded state (a fallback budget because config.json
+// was missing/unparseable, or rune-clamp-only mode because tokenizer.json could not
+// be loaded) so the caller can warn without disabling the provider.
+func newTokenTruncator(modelDir string) (*tokenTruncator, error) {
+	budget, budgetErr := readTokenBudget(modelDir)
+	t := &tokenTruncator{budget: budget, clamp: budget * runeClampBudgetFactor}
+
+	tkBytes, err := os.ReadFile(filepath.Join(modelDir, "tokenizer.json"))
+	if err != nil {
+		return t, fmt.Errorf("token truncation degraded to rune clamp: read tokenizer.json: %w", err)
+	}
+	tk, err := hftokenizer.NewFromContent(nil, tkBytes)
+	if err != nil {
+		return t, fmt.Errorf("token truncation degraded to rune clamp: parse tokenizer.json: %w", err)
+	}
+	// Encode without special tokens so every returned span is a real byte span into
+	// the original text; the two reserved positions cover the [CLS]/[SEP] the pipeline
+	// adds later. IncludeSpans is required for EncodeWithAnnotations to populate Spans.
+	if err := tk.With(api.EncodeOptions{IncludeSpans: true, AddSpecialTokens: false}); err != nil {
+		return t, fmt.Errorf("token truncation degraded to rune clamp: configure tokenizer: %w", err)
+	}
+	t.tk = tk
+
+	if budgetErr != nil {
+		return t, fmt.Errorf("using fallback token budget %d: %w", budget, budgetErr)
+	}
+	return t, nil
+}
+
+// Truncate returns text unchanged when it fits the token budget, otherwise the
+// longest prefix that stays within budget, cut on a token (and rune) boundary.
+func (t *tokenTruncator) Truncate(text string) string {
+	if t == nil || t.budget <= 0 || text == "" {
+		return text
+	}
+	// Fast path: a WordPiece/subword token spans at least one rune (true for every
+	// registered variant — MiniLM/BGE/Jina are all WordPiece), so a rune count
+	// within budget guarantees the token count is too. Only longer texts pay for the
+	// extra tokenizer pass; tokenization is µs–ms while inference dominates. (A
+	// byte-level-BPE variant, where one rune can yield several tokens, would need
+	// this invariant revisited — but the exact-tokenize path below is always correct.)
+	if utf8.RuneCountInString(text) <= t.budget {
+		return text
+	}
+	if t.tk == nil {
+		return clampRunes(text, t.clamp)
+	}
+	enc := t.tk.EncodeWithAnnotations(text)
+	if len(enc.IDs) <= t.budget {
+		return text
+	}
+	if len(enc.Spans) < t.budget {
+		// Spans unexpectedly short (tokenizer without span support) — degrade safely.
+		return clampRunes(text, t.clamp)
+	}
+	cut := enc.Spans[t.budget-1].End
+	// cut must land strictly inside the text: we only reach here with more than
+	// budget tokens, so the budget-th token ends before the end. cut == len(text)
+	// means a degenerate (zero-width) later span — fall back to the rune clamp
+	// rather than returning the full over-budget text.
+	if cut <= 0 || cut >= len(text) {
+		return clampRunes(text, t.clamp)
+	}
+	// Defensive: token spans already align to rune boundaries, but never hand back a
+	// string split mid-rune.
+	for cut < len(text) && !utf8.RuneStart(text[cut]) {
+		cut--
+	}
+	return text[:cut]
+}
+
+// TruncateAll applies Truncate to every text. It returns the input slice
+// unchanged (no allocation) when nothing needed truncating — the common case on
+// this hot path — and otherwise a copy with the over-budget entries shortened.
+func (t *tokenTruncator) TruncateAll(texts []string) []string {
+	if t == nil || t.budget <= 0 {
+		return texts
+	}
+	var out []string
+	for i, s := range texts {
+		cut := t.Truncate(s)
+		if len(cut) == len(s) { // Truncate only ever returns s or a strict prefix.
+			continue
+		}
+		if out == nil {
+			out = make([]string, len(texts))
+			copy(out, texts)
+		}
+		out[i] = cut
+	}
+	if out == nil {
+		return texts
+	}
+	return out
+}
+
+// readTokenBudget derives the truncation budget from <modelDir>/config.json's
+// max_position_embeddings. It returns the fallback budget with a non-nil error when
+// the file is missing, unparseable, or carries an implausible value — never failing.
+func readTokenBudget(modelDir string) (int, error) {
+	raw, err := os.ReadFile(filepath.Join(modelDir, "config.json"))
+	if err != nil {
+		return fallbackTokenBudget, fmt.Errorf("read config.json: %w", err)
+	}
+	var cfg struct {
+		MaxPositionEmbeddings int `json:"max_position_embeddings"`
+	}
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return fallbackTokenBudget, fmt.Errorf("parse config.json: %w", err)
+	}
+	if cfg.MaxPositionEmbeddings <= specialTokenReserve {
+		return fallbackTokenBudget, fmt.Errorf("config.json max_position_embeddings=%d is implausible", cfg.MaxPositionEmbeddings)
+	}
+	return cfg.MaxPositionEmbeddings - specialTokenReserve, nil
+}
+
+// clampRunes returns the longest prefix of text with at most maxRunes runes, always
+// cutting on a rune boundary.
+func clampRunes(text string, maxRunes int) string {
+	if maxRunes <= 0 {
+		return ""
+	}
+	count := 0
+	for i := range text {
+		if count == maxRunes {
+			return text[:i]
+		}
+		count++
+	}
+	return text
+}

internal/embedding/truncate_test.go

@@ -0,0 +1,304 @@
+package embedding
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+// tinyWordPieceTokenizer is a minimal handcrafted BERT-style WordPiece tokenizer.json
+// (lowercasing normalizer, whitespace pre-tokenizer, ~14-entry vocab). Each
+// whitespace-separated in-vocab word encodes to exactly one token, which makes the
+// truncation cut points deterministic without a real model download.
+const tinyWordPieceTokenizer = `{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {"id": 0, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
+    {"id": 100, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
+    {"id": 101, "content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
+    {"id": 102, "content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}
+  ],
+  "normalizer": {"type": "BertNormalizer", "lowercase": true},
+  "pre_tokenizer": {"type": "BertPreTokenizer"},
+  "post_processor": null,
+  "decoder": {"type": "WordPiece", "prefix": "##"},
+  "model": {
+    "type": "WordPiece",
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": "##",
+    "max_input_chars_per_word": 100,
+    "vocab": {
+      "[PAD]": 0,
+      "hello": 1,
+      "world": 2,
+      "test": 3,
+      "[UNK]": 100,
+      "[CLS]": 101,
+      "[SEP]": 102,
+      "the": 104,
+      "a": 105,
+      "is": 106,
+      "this": 107
+    }
+  }
+}`
+
+// writeModelDir creates a temp model directory containing tokenizer.json and,
+// optionally, a config.json declaring max_position_embeddings.
+func writeModelDir(t *testing.T, maxPos int) string {
+	t.Helper()
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "tokenizer.json"), []byte(tinyWordPieceTokenizer), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if maxPos > 0 {
+		cfg := fmt.Sprintf(`{"max_position_embeddings": %d}`, maxPos)
+		if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte(cfg), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return dir
+}
+
+func TestNewTokenTruncator_BudgetFromConfig(t *testing.T) {
+	// max_position_embeddings 7 minus the two reserved special-token slots = budget 5.
+	dir := writeModelDir(t, 7)
+	tr, err := newTokenTruncator(dir)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if tr.tk == nil {
+		t.Fatal("expected a real tokenizer to load")
+	}
+	if tr.budget != 5 {
+		t.Fatalf("budget = %d, want 5", tr.budget)
+	}
+}
+
+func TestNewTokenTruncator_MissingConfigFallback(t *testing.T) {
+	dir := writeModelDir(t, 0) // no config.json
+	tr, err := newTokenTruncator(dir)
+	if err == nil {
+		t.Fatal("expected an informational error about the fallback budget")
+	}
+	if tr.tk == nil {
+		t.Fatal("tokenizer must still load even when config.json is missing")
+	}
+	if tr.budget != fallbackTokenBudget {
+		t.Fatalf("budget = %d, want fallback %d", tr.budget, fallbackTokenBudget)
+	}
+}
+
+func TestTokenTruncator_ShortTextUntouched(t *testing.T) {
+	dir := writeModelDir(t, 7) // budget 5
+	tr, err := newTokenTruncator(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// 5 runes ≤ budget 5 → fast path returns the text verbatim.
+	if got := tr.Truncate("hello"); got != "hello" {
+		t.Fatalf("Truncate(hello) = %q, want unchanged", got)
+	}
+	// 11 runes > budget but only 2 tokens ≤ budget → token-count check returns verbatim.
+	if got := tr.Truncate("hello world"); got != "hello world" {
+		t.Fatalf("Truncate(hello world) = %q, want unchanged", got)
+	}
+}
+
+func TestTokenTruncator_OverBudgetCutAtSpanBoundary(t *testing.T) {
+	dir := writeModelDir(t, 7) // budget 5
+	tr, err := newTokenTruncator(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	const input = "hello world test the a is this" // 7 in-vocab words → 7 tokens
+	if n := len(tr.tk.EncodeWithAnnotations(input).IDs); n <= tr.budget {
+		t.Fatalf("test precondition: input encodes to %d tokens, need > budget %d", n, tr.budget)
+	}
+	got := tr.Truncate(input)
+	if got != "hello world test the a" {
+		t.Fatalf("Truncate cut = %q, want %q", got, "hello world test the a")
+	}
+	if !utf8.ValidString(got) {
+		t.Fatalf("truncated text is not valid UTF-8: %q", got)
+	}
+	if n := len(tr.tk.EncodeWithAnnotations(got).IDs); n > tr.budget {
+		t.Fatalf("truncated text still encodes to %d tokens, over budget %d", n, tr.budget)
+	}
+}
+
+// TestTokenTruncator_TruncateAll covers the batch API actually wired into every
+// provider's EmbedBatch: over-budget entries are shortened, in-budget entries are
+// returned untouched, and an all-fits batch returns the input slice unmodified.
+func TestTokenTruncator_TruncateAll(t *testing.T) {
+	dir := writeModelDir(t, 7) // budget 5
+	tr, err := newTokenTruncator(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	const over = "hello world test the a is this" // 7 tokens > budget 5
+
+	out := tr.TruncateAll([]string{over, "hello", "hello world"})
+	if len(out) != 3 {
+		t.Fatalf("got %d results, want 3", len(out))
+	}
+	if len(tr.tk.EncodeWithAnnotations(out[0]).IDs) > tr.budget {
+		t.Fatalf("over-budget entry not truncated: %q", out[0])
+	}
+	if out[1] != "hello" || out[2] != "hello world" {
+		t.Fatalf("in-budget entries changed: %q, %q", out[1], out[2])
+	}
+
+	// Nothing over budget -> the exact same slice is returned (no allocation).
+	fits := []string{"hello", "world"}
+	if got := tr.TruncateAll(fits); &got[0] != &fits[0] {
+		t.Fatal("TruncateAll should return the input slice when nothing needs truncating")
+	}
+}
+
+func TestReadTokenBudget(t *testing.T) {
+	cases := []struct {
+		name       string
+		configBody string // "" means no config.json
+		want       int
+		wantErr    bool
+	}{
+		{"bert-512", `{"max_position_embeddings": 512}`, 510, false},
+		{"jina-8192", `{"max_position_embeddings": 8192}`, 8190, false},
+		{"missing", "", fallbackTokenBudget, true},
+		{"unparseable", `{not json`, fallbackTokenBudget, true},
+		{"absent-field", `{"hidden_size": 384}`, fallbackTokenBudget, true},
+		{"implausible", `{"max_position_embeddings": 1}`, fallbackTokenBudget, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			if tc.configBody != "" {
+				if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte(tc.configBody), 0o644); err != nil {
+					t.Fatal(err)
+				}
+			}
+			got, err := readTokenBudget(dir)
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("err = %v, wantErr %v", err, tc.wantErr)
+			}
+			if got != tc.want {
+				t.Fatalf("budget = %d, want %d", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestClampRunes(t *testing.T) {
+	cases := []struct {
+		name     string
+		text     string
+		maxRunes int
+		want     string
+	}{
+		{"multibyte not split", "héllo wörld", 3, "hél"},
+		{"under cap untouched", "abc", 10, "abc"},
+		{"zero cap", "abc", 0, ""},
+		{"exact cap", "abcd", 4, "abcd"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := clampRunes(tc.text, tc.maxRunes)
+			if got != tc.want {
+				t.Fatalf("clampRunes(%q, %d) = %q, want %q", tc.text, tc.maxRunes, got, tc.want)
+			}
+			if !utf8.ValidString(got) {
+				t.Fatalf("result not valid UTF-8: %q", got)
+			}
+		})
+	}
+}
+
+// TestHugotProvider_LiveTruncatesOverBudgetInput is the end-to-end regression
+// for the shape-mismatch crash: an input well past MiniLM's 512-token window
+// used to abort the pipeline. With truncation it must embed to a real 384-dim
+// vector. Gated on a real cached model — set GORTEX_TEST_LIVE_MODELS=1.
+func TestHugotProvider_LiveTruncatesOverBudgetInput(t *testing.T) {
+	if os.Getenv("GORTEX_TEST_LIVE_MODELS") != "1" {
+		t.Skip("set GORTEX_TEST_LIVE_MODELS=1 to run against the real MiniLM model")
+	}
+	prov, err := newHugotProvider()
+	if err != nil {
+		t.Skipf("MiniLM model unavailable: %v", err)
+	}
+	defer prov.Close()
+
+	// ~120 repetitions of a 9-token sentence ≈ 1000+ tokens — comfortably past
+	// the 512-token window that used to crash the GO tokenizer path.
+	over := strings.Repeat("the quick brown fox jumps over the lazy dog ", 120)
+
+	vec, err := prov.Embed(context.Background(), over)
+	if err != nil {
+		t.Fatalf("embedding an over-budget input failed (truncation regression): %v", err)
+	}
+	if len(vec) != prov.Dimensions() {
+		t.Fatalf("got a %d-dim vector, want %d", len(vec), prov.Dimensions())
+	}
+	nonZero := false
+	for _, v := range vec {
+		if v != 0 {
+			nonZero = true
+			break
+		}
+	}
+	if !nonZero {
+		t.Fatal("embedding vector is all zeros")
+	}
+
+	// A batch mixing an over-budget input with a short one must also succeed.
+	vecs, err := prov.EmbedBatch(context.Background(), []string{over, "short input"})
+	if err != nil {
+		t.Fatalf("mixed batch failed: %v", err)
+	}
+	if len(vecs) != 2 || len(vecs[0]) != prov.Dimensions() || len(vecs[1]) != prov.Dimensions() {
+		t.Fatalf("mixed batch produced wrong shapes: %d vectors", len(vecs))
+	}
+}
+
+func TestNewTokenTruncator_CorruptTokenizerRuneClamp(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "tokenizer.json"), []byte("{ not a tokenizer"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"max_position_embeddings": 12}`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	tr, err := newTokenTruncator(dir)
+	if err == nil {
+		t.Fatal("expected an error when tokenizer.json is corrupt")
+	}
+	if tr == nil {
+		t.Fatal("truncator must be non-nil even on tokenizer failure")
+	}
+	if tr.tk != nil {
+		t.Fatal("tk must be nil in rune-clamp mode")
+	}
+	// clamp must equal the budget (not a multiple): a token spans >= 1 rune, so
+	// budget runes bound the token count to budget — a looser cap would not.
+	if tr.budget != 10 || tr.clamp != 10 {
+		t.Fatalf("budget=%d clamp=%d, want 10/10", tr.budget, tr.clamp)
+	}
+	// A long multibyte text is clamped (not split mid-rune) rather than passed through.
+	long := ""
+	for i := 0; i < 100; i++ {
+		long += "café "
+	}
+	got := tr.Truncate(long)
+	if utf8.RuneCountInString(got) > tr.clamp {
+		t.Fatalf("clamped rune count %d exceeds clamp %d", utf8.RuneCountInString(got), tr.clamp)
+	}
+	if !utf8.ValidString(got) {
+		t.Fatalf("clamped text not valid UTF-8")
+	}
+}

internal/embedding/validate.go

@@ -0,0 +1,35 @@
+package embedding
+
+import "fmt"
+
+// validateBatch checks that an embedding backend returned exactly one vector per
+// input, that no vector is empty, and that every vector has the expected width.
+//
+// dims is the expected width. Pass a provider's known dimension (e.g. a Hugot
+// model's 384) for a strict check; pass 0 when the width is not yet known — an
+// API provider learns its width lazily from the first response — and the width
+// is taken from the first returned vector so the batch is still checked for
+// internal consistency (a ragged batch) without asserting an absolute size.
+//
+// A violation returns a descriptive error naming the offending index so a
+// provider regression that returns nil, short, or mis-counted vectors surfaces
+// as a loud, attributable failure instead of a silently empty or mis-dimensioned
+// index.
+func validateBatch(name string, texts []string, vecs [][]float32, dims int) error {
+	if len(vecs) != len(texts) {
+		return fmt.Errorf("%s returned %d vectors for %d inputs", name, len(vecs), len(texts))
+	}
+	expected := dims
+	if expected <= 0 && len(vecs) > 0 {
+		expected = len(vecs[0])
+	}
+	for i, v := range vecs {
+		if len(v) == 0 {
+			return fmt.Errorf("%s returned an empty vector at index %d of %d", name, i, len(vecs))
+		}
+		if expected > 0 && len(v) != expected {
+			return fmt.Errorf("%s returned a width-%d vector at index %d, want %d", name, len(v), i, expected)
+		}
+	}
+	return nil
+}

internal/embedding/validate_test.go

@@ -0,0 +1,33 @@
+package embedding
+
+import "testing"
+
+func TestValidateBatch(t *testing.T) {
+	vec := func(n int) []float32 { return make([]float32, n) }
+
+	cases := []struct {
+		name    string
+		texts   []string
+		vecs    [][]float32
+		dims    int
+		wantErr bool
+	}{
+		{"ok strict width", []string{"a", "b"}, [][]float32{vec(384), vec(384)}, 384, false},
+		{"ok empty batch", []string{}, [][]float32{}, 384, false},
+		{"ok lazy width consistent", []string{"a", "b"}, [][]float32{vec(768), vec(768)}, 0, false},
+		{"count mismatch short", []string{"a", "b"}, [][]float32{vec(384)}, 384, true},
+		{"count mismatch extra", []string{"a"}, [][]float32{vec(384), vec(384)}, 384, true},
+		{"nil vector", []string{"a", "b"}, [][]float32{vec(384), nil}, 384, true},
+		{"empty vector", []string{"a"}, [][]float32{{}}, 384, true},
+		{"wrong strict width", []string{"a"}, [][]float32{vec(100)}, 384, true},
+		{"ragged lazy width", []string{"a", "b"}, [][]float32{vec(768), vec(384)}, 0, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := validateBatch("test", tc.texts, tc.vecs, tc.dims)
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("validateBatch err = %v, wantErr %v", err, tc.wantErr)
+			}
+		})
+	}
+}

internal/embedding/variant_test.go

@@ -0,0 +1,100 @@
+package embedding
+
+import (
+	"strings"
+	"testing"
+)
+
+// TestNewProviderFromConfig_EmptyVariantUsesDefault asserts that a
+// "local" provider with no variant takes the default-selection path
+// (NewLocalProvider), never the variant lookup. We can't assert the
+// concrete returned type without triggering a model download, so we
+// assert the negative: an empty variant must NOT produce the
+// "unknown hugot variant" error that only the variant branch can emit.
+func TestNewProviderFromConfig_EmptyVariantUsesDefault(t *testing.T) {
+	// Keep this hermetic and offline. The "local" path runs the real
+	// backend chain (ONNX → GoMLX → Hugot → static); with a cold cache
+	// Hugot would download MiniLM over the network — slow, flaky, and
+	// racy under -race inside go-huggingface's parallel downloader. Point
+	// the model cache at an empty temp dir and disable downloads so the
+	// chain deterministically falls back to the static provider, which is
+	// exactly the no-error, non-nil result this test asserts.
+	t.Setenv("XDG_DATA_HOME", t.TempDir())
+	t.Setenv(embeddingOfflineEnv, "1")
+
+	p, err := NewProviderFromConfig(ProviderConfig{Provider: "local", Variant: ""})
+	if err != nil {
+		// NewLocalProvider falls back to the static provider if no
+		// transformer backend is available, so it must never error.
+		t.Fatalf("local provider with empty variant errored: %v", err)
+	}
+	if p == nil {
+		t.Fatalf("local provider with empty variant returned nil provider")
+	}
+	_ = p.Close()
+}
+
+// TestNewProviderFromConfig_NamedVariantRoutesToHugot asserts that a
+// non-empty variant routes through NewHugotProviderWithVariant. An
+// unknown variant name lets us prove the routing without a download:
+// only the variant branch produces the "unknown hugot variant" error;
+// NewLocalProvider would silently fall back to static and never raise
+// it. The error must also enumerate the known variants.
+func TestNewProviderFromConfig_NamedVariantRoutesToHugot(t *testing.T) {
+	_, err := NewProviderFromConfig(ProviderConfig{
+		Provider: "local",
+		Variant:  "definitely-not-a-real-variant",
+	})
+	if err == nil {
+		t.Fatalf("expected an error for an unknown variant, got nil — routing did not reach the variant branch")
+	}
+	if !strings.Contains(err.Error(), "unknown hugot variant") {
+		t.Fatalf("error %q does not name the variant branch — routing likely went through NewLocalProvider", err)
+	}
+	// The error must list the real variants, proving KnownHugotVariants
+	// is the routing table.
+	for _, name := range KnownHugotVariants() {
+		if !strings.Contains(err.Error(), name) {
+			t.Fatalf("unknown-variant error must enumerate known variants; missing %q in %q", name, err.Error())
+		}
+	}
+}
+
+// TestNewProviderFromConfig_VariantIgnoredForNonLocal asserts the
+// variant is honoured only for the "local" provider — a static provider
+// with a bogus variant must still construct fine (the variant is
+// ignored), so an existing static config can never break by carrying a
+// stray variant value.
+func TestNewProviderFromConfig_VariantIgnoredForNonLocal(t *testing.T) {
+	p, err := NewProviderFromConfig(ProviderConfig{
+		Provider: "static",
+		Variant:  "definitely-not-a-real-variant",
+	})
+	if err != nil {
+		t.Fatalf("static provider must ignore the variant, got error: %v", err)
+	}
+	if p == nil {
+		t.Fatalf("static provider returned nil")
+	}
+	_ = p.Close()
+}
+
+// TestKnownHugotVariants_AllLookupable asserts every name returned by
+// KnownHugotVariants resolves through LookupHugotVariant — the variant
+// table and the public enumeration can never drift apart, so a name a
+// caller reads from KnownHugotVariants is always a valid variant arg.
+func TestKnownHugotVariants_AllLookupable(t *testing.T) {
+	names := KnownHugotVariants()
+	if len(names) == 0 {
+		t.Fatal("KnownHugotVariants returned no variants")
+	}
+	for _, name := range names {
+		if _, ok := LookupHugotVariant(name); !ok {
+			t.Fatalf("KnownHugotVariants lists %q but LookupHugotVariant rejects it", name)
+		}
+	}
+	// The default variant must itself be a known, lookupable name.
+	if _, ok := LookupHugotVariant(DefaultHugotVariant); !ok {
+		t.Fatalf("DefaultHugotVariant %q is not a known variant", DefaultHugotVariant)
+	}
+}

internal/embedding/wordpiece.go

@@ -0,0 +1,271 @@
+package embedding
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/unicode/norm"
+)
+
+// wordPieceTokenizer is a minimal, dependency-free implementation of the
+// HuggingFace fast-tokenizer trio used by BERT-style vocabularies:
+// BertNormalizer → BertPreTokenizer → WordPiece. It loads the standard
+// tokenizer.json and reproduces the exact token-ID sequences of the
+// reference implementation for that configuration (verified by golden
+// tests against the upstream tokenizer).
+//
+// Scope: only the configuration the bundled static code-embedding model
+// uses — BertNormalizer{clean_text, handle_chinese_chars, lowercase,
+// strip_accents:null} + BertPreTokenizer + WordPiece{"##"} — is
+// implemented. Loading a tokenizer.json with a different model type
+// fails loudly rather than mis-tokenizing quietly.
+type wordPieceTokenizer struct {
+	vocab            map[string]int
+	unkID            int
+	contPrefix       string
+	maxWordChars     int
+	lowercase        bool
+	stripAccents     bool
+	cleanText        bool
+	handleCJK        bool
+}
+
+// tokenizerJSON mirrors just the fields of tokenizer.json this
+// implementation consumes.
+type tokenizerJSON struct {
+	Normalizer *struct {
+		Type         string `json:"type"`
+		CleanText    *bool  `json:"clean_text"`
+		HandleCJK    *bool  `json:"handle_chinese_chars"`
+		StripAccents *bool  `json:"strip_accents"`
+		Lowercase    *bool  `json:"lowercase"`
+	} `json:"normalizer"`
+	PreTokenizer *struct {
+		Type string `json:"type"`
+	} `json:"pre_tokenizer"`
+	Model struct {
+		Type                    string         `json:"type"`
+		UnkToken                string         `json:"unk_token"`
+		ContinuingSubwordPrefix string         `json:"continuing_subword_prefix"`
+		MaxInputCharsPerWord    *int           `json:"max_input_chars_per_word"`
+		Vocab                   map[string]int `json:"vocab"`
+	} `json:"model"`
+}
+
+// loadWordPieceTokenizer parses a tokenizer.json from disk.
+func loadWordPieceTokenizer(path string) (*wordPieceTokenizer, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read tokenizer: %w", err)
+	}
+	var tj tokenizerJSON
+	if err := json.Unmarshal(raw, &tj); err != nil {
+		return nil, fmt.Errorf("parse tokenizer: %w", err)
+	}
+	if tj.Model.Type != "WordPiece" {
+		return nil, fmt.Errorf("unsupported tokenizer model %q (want WordPiece)", tj.Model.Type)
+	}
+	if len(tj.Model.Vocab) == 0 {
+		return nil, fmt.Errorf("tokenizer vocab is empty")
+	}
+	t := &wordPieceTokenizer{
+		vocab:        tj.Model.Vocab,
+		contPrefix:   tj.Model.ContinuingSubwordPrefix,
+		maxWordChars: 100,
+		// BertNormalizer defaults per the reference implementation.
+		cleanText: true,
+		handleCJK: true,
+	}
+	if t.contPrefix == "" {
+		t.contPrefix = "##"
+	}
+	if tj.Model.MaxInputCharsPerWord != nil && *tj.Model.MaxInputCharsPerWord > 0 {
+		t.maxWordChars = *tj.Model.MaxInputCharsPerWord
+	}
+	unk := tj.Model.UnkToken
+	if unk == "" {
+		unk = "[UNK]"
+	}
+	id, ok := t.vocab[unk]
+	if !ok {
+		return nil, fmt.Errorf("unk token %q not in vocab", unk)
+	}
+	t.unkID = id
+	if n := tj.Normalizer; n != nil {
+		if n.CleanText != nil {
+			t.cleanText = *n.CleanText
+		}
+		if n.HandleCJK != nil {
+			t.handleCJK = *n.HandleCJK
+		}
+		if n.Lowercase != nil {
+			t.lowercase = *n.Lowercase
+		}
+		// strip_accents:null means "follow lowercase" in the reference
+		// implementation; an explicit value wins.
+		if n.StripAccents != nil {
+			t.stripAccents = *n.StripAccents
+		} else {
+			t.stripAccents = t.lowercase
+		}
+	}
+	return t, nil
+}
+
+// Encode returns the WordPiece token IDs for text. No special tokens
+// are added (the model's post_processor is null).
+func (t *wordPieceTokenizer) Encode(text string) []int {
+	var ids []int
+	for _, word := range t.preTokenize(t.normalize(text)) {
+		ids = t.wordPiece(word, ids)
+	}
+	return ids
+}
+
+// normalize applies BertNormalizer: control-char cleanup, CJK padding,
+// accent stripping (NFD, drop Mn), lowercasing.
+func (t *wordPieceTokenizer) normalize(s string) string {
+	var b strings.Builder
+	b.Grow(len(s) + 8)
+	for _, r := range s {
+		if t.cleanText {
+			if r == 0 || r == 0xFFFD || isBertControl(r) {
+				continue
+			}
+			if isBertWhitespace(r) {
+				b.WriteByte(' ')
+				continue
+			}
+		}
+		if t.handleCJK && isCJK(r) {
+			b.WriteByte(' ')
+			b.WriteRune(r)
+			b.WriteByte(' ')
+			continue
+		}
+		b.WriteRune(r)
+	}
+	out := b.String()
+	if t.stripAccents {
+		decomposed := norm.NFD.String(out)
+		var sb strings.Builder
+		sb.Grow(len(decomposed))
+		for _, r := range decomposed {
+			if unicode.Is(unicode.Mn, r) {
+				continue
+			}
+			sb.WriteRune(r)
+		}
+		out = sb.String()
+	}
+	if t.lowercase {
+		out = strings.ToLower(out)
+	}
+	return out
+}
+
+// preTokenize applies BertPreTokenizer: split on whitespace, then
+// isolate each punctuation rune as its own token.
+func (t *wordPieceTokenizer) preTokenize(s string) []string {
+	var words []string
+	var cur strings.Builder
+	flush := func() {
+		if cur.Len() > 0 {
+			words = append(words, cur.String())
+			cur.Reset()
+		}
+	}
+	for _, r := range s {
+		switch {
+		case isBertWhitespace(r):
+			flush()
+		case isBertPunct(r):
+			flush()
+			words = append(words, string(r))
+		default:
+			cur.WriteRune(r)
+		}
+	}
+	flush()
+	return words
+}
+
+// wordPiece runs the greedy longest-match-first sub-word split for one
+// word, appending IDs to dst.
+func (t *wordPieceTokenizer) wordPiece(word string, dst []int) []int {
+	runes := []rune(word)
+	if len(runes) > t.maxWordChars {
+		return append(dst, t.unkID)
+	}
+	start := 0
+	var pieces []int
+	for start < len(runes) {
+		end := len(runes)
+		id := -1
+		for end > start {
+			sub := string(runes[start:end])
+			if start > 0 {
+				sub = t.contPrefix + sub
+			}
+			if v, ok := t.vocab[sub]; ok {
+				id = v
+				break
+			}
+			end--
+		}
+		if id < 0 {
+			// No piece matched: the whole word becomes UNK, per the
+			// reference implementation.
+			return append(dst, t.unkID)
+		}
+		pieces = append(pieces, id)
+		start = end
+	}
+	return append(dst, pieces...)
+}
+
+// isBertControl mirrors the reference _is_control: category Cc/Cf,
+// except tab / newline / carriage-return which count as whitespace.
+func isBertControl(r rune) bool {
+	if r == '\t' || r == '\n' || r == '\r' {
+		return false
+	}
+	return unicode.Is(unicode.Cc, r) || unicode.Is(unicode.Cf, r)
+}
+
+// isBertWhitespace mirrors the reference _is_whitespace.
+func isBertWhitespace(r rune) bool {
+	switch r {
+	case ' ', '\t', '\n', '\r':
+		return true
+	}
+	return unicode.Is(unicode.Zs, r)
+}
+
+// isBertPunct mirrors the reference _is_punctuation: the four ASCII
+// symbol ranges plus every Unicode P* category rune.
+func isBertPunct(r rune) bool {
+	if (r >= 33 && r <= 47) || (r >= 58 && r <= 64) || (r >= 91 && r <= 96) || (r >= 123 && r <= 126) {
+		return true
+	}
+	return unicode.IsPunct(r)
+}
+
+// isCJK mirrors the reference _is_chinese_char CJK block test.
+func isCJK(r rune) bool {
+	switch {
+	case r >= 0x4E00 && r <= 0x9FFF,
+		r >= 0x3400 && r <= 0x4DBF,
+		r >= 0x20000 && r <= 0x2A6DF,
+		r >= 0x2A700 && r <= 0x2B73F,
+		r >= 0x2B740 && r <= 0x2B81F,
+		r >= 0x2B820 && r <= 0x2CEAF,
+		r >= 0xF900 && r <= 0xFAFF,
+		r >= 0x2F800 && r <= 0x2FA1F:
+		return true
+	}
+	return false
+}