chore: import upstream snapshot with attribution

internal/embedding/static_test.go

@@ -0,0 +1,171 @@
+package embedding
+
+import (
+	"context"
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// cosine returns the cosine similarity of two equal-length vectors.
+// Both inputs from StaticProvider are L2-normalized, so this is just
+// the dot product — but the explicit norm keeps the helper correct
+// for any caller.
+func cosine(a, b []float32) float64 {
+	var dot, na, nb float64
+	for i := range a {
+		dot += float64(a[i]) * float64(b[i])
+		na += float64(a[i]) * float64(a[i])
+		nb += float64(b[i]) * float64(b[i])
+	}
+	if na == 0 || nb == 0 {
+		return 0
+	}
+	return dot / (math.Sqrt(na) * math.Sqrt(nb))
+}
+
+// TestStaticProvider_LoadsBakedTable asserts the embedded GloVe table
+// loads and reports a plausible dimensionality and vocabulary size.
+func TestStaticProvider_LoadsBakedTable(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	defer p.Close()
+
+	// The baked table is GloVe 6B 50d. Dimensions() must reflect the
+	// loaded table, not the 50 hardcoded in the constructor before
+	// loadVectors runs.
+	assert.Equal(t, 50, p.Dimensions(), "GloVe 6B baked table is 50-dimensional")
+	assert.Greater(t, len(p.vectors), 10_000, "baked table should carry the top-~20k vocabulary")
+}
+
+// TestStaticProvider_SemanticRanking asserts that semantically related
+// code tokens land closer in vector space than unrelated ones — the
+// core property that makes static embeddings a useful fusion signal.
+//
+// The word pairs are drawn from the baked GloVe top-20k vocabulary
+// (verb pairs like validate/delete fall below GloVe 6B's frequency
+// cutoff and are absent — the tokenizer would lower them to a zero
+// vector, so the test uses present near-synonyms instead).
+func TestStaticProvider_SemanticRanking(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	defer p.Close()
+	ctx := context.Background()
+
+	embed := func(s string) []float32 {
+		v, err := p.Embed(ctx, s)
+		require.NoError(t, err)
+		require.NotEmpty(t, v)
+		// A non-zero vector confirms the token was found in-vocab; a
+		// miss would average to all-zero and make the comparison moot.
+		nonZero := false
+		for _, x := range v {
+			if x != 0 {
+				nonZero = true
+				break
+			}
+		}
+		require.True(t, nonZero, "token %q must be in the baked vocabulary", s)
+		return v
+	}
+
+	cases := []struct {
+		related, synonym, unrelated string
+	}{
+		{"check", "verify", "banana"},
+		{"request", "response", "banana"},
+		{"file", "directory", "mountain"},
+	}
+	for _, c := range cases {
+		base := embed(c.related)
+		syn := embed(c.synonym)
+		unrel := embed(c.unrelated)
+		simRelated := cosine(base, syn)
+		simUnrelated := cosine(base, unrel)
+		assert.Greater(t, simRelated, simUnrelated,
+			"%q should be closer to %q (%.3f) than to %q (%.3f)",
+			c.related, c.synonym, simRelated, c.unrelated, simUnrelated)
+	}
+}
+
+// TestStaticProvider_EmbedBatchRoundTrip asserts EmbedBatch returns one
+// vector per input in order, matching the per-item Embed result.
+func TestStaticProvider_EmbedBatchRoundTrip(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	defer p.Close()
+	ctx := context.Background()
+
+	inputs := []string{"validate token", "delete user", "parse json"}
+	batch, err := p.EmbedBatch(ctx, inputs)
+	require.NoError(t, err)
+	require.Len(t, batch, len(inputs))
+
+	for i, in := range inputs {
+		want, err := p.Embed(ctx, in)
+		require.NoError(t, err)
+		require.Len(t, batch[i], p.Dimensions())
+		assert.Equal(t, want, batch[i], "EmbedBatch[%d] must match Embed(%q)", i, in)
+	}
+}
+
+// TestStaticProvider_EmbedBatchEmpty asserts the empty-input edge case
+// returns an empty (non-nil-shaped) slice without error.
+func TestStaticProvider_EmbedBatchEmpty(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	defer p.Close()
+
+	out, err := p.EmbedBatch(context.Background(), nil)
+	require.NoError(t, err)
+	assert.Empty(t, out)
+}
+
+// TestTokenizeForEmbedding_Splitters covers every boundary the
+// tokenizer must split on: camelCase, underscore, dot, slash, and
+// mixed identifiers — plus the single-char drop rule.
+func TestTokenizeForEmbedding_Splitters(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  []string
+	}{
+		{"camelCase", "validateUserToken", []string{"validate", "user", "token"}},
+		{"underscore", "get_user_by_id", []string{"get", "user", "by", "id"}},
+		{"dotPath", "config.search.weights", []string{"config", "search", "weights"}},
+		{"slashPath", "internal/embedding/static.go", []string{"internal", "embedding", "static", "go"}},
+		{"mixed", "internal/auth.checkAccessToken", []string{"internal", "auth", "check", "access", "token"}},
+		{"singleCharsDropped", "a b c d", nil},
+		{"leadingUpper", "HTTPHandler", []string{"httphandler"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tokenizeForEmbedding(tt.input)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+// TestStaticProvider_SetVectorsInjection asserts SetVectors swaps the
+// table and re-derives Dimensions from the injected vectors — the
+// deterministic-injection path used by other tests.
+func TestStaticProvider_SetVectorsInjection(t *testing.T) {
+	p, err := NewStaticProvider()
+	require.NoError(t, err)
+	defer p.Close()
+
+	p.SetVectors(map[string][]float32{
+		"alpha": {1, 0, 0, 0},
+		"beta":  {0, 1, 0, 0},
+	})
+	assert.Equal(t, 4, p.Dimensions(), "Dimensions must follow the injected vector width")
+
+	vec, err := p.Embed(context.Background(), "alpha")
+	require.NoError(t, err)
+	require.Len(t, vec, 4)
+	// Single in-vocab token: averaging then normalizing a unit basis
+	// vector returns it unchanged.
+	assert.InDelta(t, 1.0, vec[0], 1e-6)
+}