/* * test_semantic.c — Unit tests for semantic.c (pure functions). * * Covers: tokenize, cosine, normalize, vec_add_scaled, random_index, * proximity, diffuse, corpus lifecycle, get_config. */ #include "test_framework.h" #include #include #include #include #include /* ── Tokenize ────────────────────────────────────────────────────── */ TEST(sem_tokenize_camel) { char *tokens[32]; int n = cbm_sem_tokenize("parseUserInput", tokens, 32); ASSERT_GTE(n, 3); ASSERT_STR_EQ(tokens[0], "parse"); ASSERT_STR_EQ(tokens[1], "user"); ASSERT_STR_EQ(tokens[2], "input"); for (int i = 0; i < n; i++) free(tokens[i]); PASS(); } TEST(sem_tokenize_snake) { char *tokens[32]; int n = cbm_sem_tokenize("handle_http_request", tokens, 32); ASSERT_GTE(n, 3); ASSERT_STR_EQ(tokens[0], "handle"); ASSERT_STR_EQ(tokens[1], "http"); ASSERT_STR_EQ(tokens[2], "request"); for (int i = 0; i < n; i++) free(tokens[i]); PASS(); } TEST(sem_tokenize_dot) { char *tokens[32]; int n = cbm_sem_tokenize("net.http.client", tokens, 32); ASSERT_GTE(n, 3); ASSERT_STR_EQ(tokens[0], "net"); ASSERT_STR_EQ(tokens[1], "http"); ASSERT_STR_EQ(tokens[2], "client"); for (int i = 0; i < n; i++) free(tokens[i]); PASS(); } TEST(sem_tokenize_null) { int n = cbm_sem_tokenize(NULL, NULL, 0); ASSERT_EQ(n, 0); PASS(); } TEST(sem_tokenize_max_out) { char *tokens[3]; int n = cbm_sem_tokenize("a_b_c_d_e_f_g", tokens, 3); ASSERT_EQ(n, 3); for (int i = 0; i < n; i++) free(tokens[i]); PASS(); } TEST(sem_tokenize_abbrev_expansion) { char *tokens[32]; int n = cbm_sem_tokenize("getCtxErrMsg", tokens, 32); /* get, ctx, context, err, error, msg, message */ ASSERT_GTE(n, 4); bool has_ctx = false, has_context = false, has_err = false, has_error = false; for (int i = 0; i < n; i++) { if (strcmp(tokens[i], "ctx") == 0) has_ctx = true; if (strcmp(tokens[i], "context") == 0) has_context = true; if (strcmp(tokens[i], "err") == 0) has_err = true; if (strcmp(tokens[i], "error") == 0) has_error = true; } ASSERT_TRUE(has_ctx && has_context && has_err && has_error); for (int i = 0; i < n; i++) free(tokens[i]); PASS(); } /* ── Cosine similarity ───────────────────────────────────────────── */ static void fill_vec(cbm_sem_vec_t *v, float val) { for (int i = 0; i < CBM_SEM_DIM; i++) v->v[i] = val; } TEST(sem_cosine_identical) { cbm_sem_vec_t a, b; fill_vec(&a, 0.5f); fill_vec(&b, 0.5f); float sim = cbm_sem_cosine(&a, &b); ASSERT_FLOAT_EQ(sim, 1.0f, 0.001f); PASS(); } TEST(sem_cosine_orthogonal) { cbm_sem_vec_t a, b; memset(&a, 0, sizeof(a)); memset(&b, 0, sizeof(b)); a.v[0] = 1.0f; b.v[1] = 1.0f; float sim = cbm_sem_cosine(&a, &b); ASSERT_FLOAT_EQ(sim, 0.0f, 0.001f); PASS(); } TEST(sem_cosine_zero_vector) { cbm_sem_vec_t a, b; memset(&a, 0, sizeof(a)); fill_vec(&b, 1.0f); float sim = cbm_sem_cosine(&a, &b); ASSERT_FLOAT_EQ(sim, 0.0f, 0.001f); PASS(); } TEST(sem_cosine_negative) { cbm_sem_vec_t a, b; memset(&a, 0, sizeof(a)); memset(&b, 0, sizeof(b)); a.v[0] = 1.0f; b.v[0] = -1.0f; float sim = cbm_sem_cosine(&a, &b); ASSERT_FLOAT_EQ(sim, -1.0f, 0.001f); PASS(); } TEST(sem_cosine_null) { ASSERT_FLOAT_EQ(cbm_sem_cosine(NULL, NULL), 0.0f, 0.001f); PASS(); } /* ── Normalize ───────────────────────────────────────────────────── */ TEST(sem_normalize_unit) { cbm_sem_vec_t v; memset(&v, 0, sizeof(v)); v.v[0] = 1.0f; cbm_sem_normalize(&v); ASSERT_FLOAT_EQ(cbm_sem_cosine(&v, &v), 1.0f, 0.001f); PASS(); } TEST(sem_normalize_scales) { cbm_sem_vec_t v; fill_vec(&v, 2.0f); cbm_sem_normalize(&v); float mag_sq = 0.0f; for (int i = 0; i < CBM_SEM_DIM; i++) mag_sq += v.v[i] * v.v[i]; float mag = sqrtf(mag_sq); ASSERT_FLOAT_EQ(mag, 1.0f, 0.01f); PASS(); } TEST(sem_normalize_zero) { cbm_sem_vec_t v; memset(&v, 0, sizeof(v)); cbm_sem_normalize(&v); /* Should remain zero (no division by zero) */ PASS(); } TEST(sem_normalize_null) { cbm_sem_normalize(NULL); /* should not crash */ PASS(); } /* ── Vec add scaled ──────────────────────────────────────────────── */ TEST(sem_vec_add_scaled_basic) { cbm_sem_vec_t dst; memset(&dst, 0, sizeof(dst)); cbm_sem_vec_t src; fill_vec(&src, 1.0f); cbm_sem_vec_add_scaled(&dst, &src, 0.5f); ASSERT_FLOAT_EQ(dst.v[0], 0.5f, 0.001f); ASSERT_FLOAT_EQ(dst.v[CBM_SEM_DIM - 1], 0.5f, 0.001f); PASS(); } TEST(sem_vec_add_scaled_null) { cbm_sem_vec_t v; fill_vec(&v, 1.0f); cbm_sem_vec_add_scaled(NULL, &v, 1.0f); /* should not crash */ cbm_sem_vec_add_scaled(&v, NULL, 1.0f); /* should not crash */ PASS(); } /* ── Random index ────────────────────────────────────────────────── */ TEST(sem_random_index_deterministic) { cbm_sem_vec_t a, b; cbm_sem_random_index("hello", &a); cbm_sem_random_index("hello", &b); ASSERT_FLOAT_EQ(cbm_sem_cosine(&a, &b), 1.0f, 0.001f); PASS(); } TEST(sem_random_index_different_tokens) { cbm_sem_vec_t a, b; cbm_sem_random_index("function", &a); cbm_sem_random_index("variable", &b); /* Different tokens should produce different vectors */ float sim = cbm_sem_cosine(&a, &b); ASSERT_TRUE(sim < 1.0f - 1e-6f); PASS(); } TEST(sem_random_index_null) { cbm_sem_vec_t v; memset(&v, 0, sizeof(v)); cbm_sem_random_index(NULL, &v); /* Should produce zero vector for NULL token */ for (int i = 0; i < CBM_SEM_DIM; i++) { ASSERT_FLOAT_EQ(v.v[i], 0.0f, 0.001f); } PASS(); } /* ── Proximity ───────────────────────────────────────────────────── */ TEST(sem_proximity_same_file) { float p = cbm_sem_proximity("src/main.c", "src/main.c"); ASSERT_FLOAT_EQ(p, 1.1f, 0.01f); /* CBM_SEM_UNIT_POS + CBM_SEM_PROX_MAX_BOOST */ PASS(); } TEST(sem_proximity_same_dir) { /* Files sharing 1 of 2 directory components: ratio = 0.5 → 1.0 + 0.5*0.10 = 1.05 */ float p = cbm_sem_proximity("src/core/a.c", "src/io/b.c"); ASSERT_TRUE(p > 1.0f && p < 1.10f); PASS(); } TEST(sem_proximity_different_paths) { float p = cbm_sem_proximity("src/foo/a.c", "tests/bar/b.c"); ASSERT_FLOAT_EQ(p, 1.0f, 0.01f); PASS(); } TEST(sem_proximity_null) { ASSERT_FLOAT_EQ(cbm_sem_proximity(NULL, "foo.c"), 1.0f, 0.01f); ASSERT_FLOAT_EQ(cbm_sem_proximity("foo.c", NULL), 1.0f, 0.01f); PASS(); } /* ── Diffuse ─────────────────────────────────────────────────────── */ TEST(sem_diffuse_zero_neighbors) { cbm_sem_vec_t v; fill_vec(&v, 0.5f); cbm_sem_diffuse(&v, NULL, 0, 0.3f); /* With zero neighbors, vector should be unchanged */ ASSERT_FLOAT_EQ(v.v[0], 0.5f, 0.001f); PASS(); } TEST(sem_diffuse_single_neighbor) { cbm_sem_vec_t v; memset(&v, 0, sizeof(v)); v.v[0] = 0.5f; v.v[1] = 0.5f; cbm_sem_normalize(&v); /* unit-length input */ cbm_sem_vec_t nb; memset(&nb, 0, sizeof(nb)); nb.v[0] = 1.0f; cbm_sem_normalize(&nb); cbm_sem_diffuse(&v, &nb, 1, 0.3f); /* After diffuse+normalize, result should still be unit-length */ float mag_sq = 0.0f; for (int i = 0; i < CBM_SEM_DIM; i++) mag_sq += v.v[i] * v.v[i]; ASSERT_FLOAT_EQ(sqrtf(mag_sq), 1.0f, 0.01f); /* Component 0 should be pulled toward neighbor's strong dim-0 */ ASSERT_TRUE(v.v[0] > 0.0f); PASS(); } /* ── Corpus lifecycle ────────────────────────────────────────────── */ TEST(sem_corpus_new_free) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); ASSERT_EQ(cbm_sem_corpus_doc_count(c), 0); ASSERT_EQ(cbm_sem_corpus_token_count(c), 0); cbm_sem_corpus_free(c); PASS(); } TEST(sem_corpus_add_one_doc) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); const char *tokens[] = {"parse", "user", "input"}; cbm_sem_corpus_add_doc(c, tokens, 3); ASSERT_EQ(cbm_sem_corpus_doc_count(c), 1); ASSERT_TRUE(cbm_sem_corpus_token_count(c) > 0); cbm_sem_corpus_free(c); PASS(); } TEST(sem_corpus_idf) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); const char *doc1[] = {"a", "b", "c"}; const char *doc2[] = {"a", "d", "e"}; cbm_sem_corpus_add_doc(c, doc1, 3); cbm_sem_corpus_add_doc(c, doc2, 3); /* IDF for "a" (appears in 2 docs): log(2/2) = log(1) = 0 */ float idf_a = cbm_sem_corpus_idf(c, "a"); ASSERT_TRUE(idf_a < 0.01f); /* IDF for "b" (appears in 1 doc): log(2/1) > 0 */ float idf_b = cbm_sem_corpus_idf(c, "b"); ASSERT_TRUE(idf_b > 0.0f); cbm_sem_corpus_free(c); PASS(); } TEST(sem_corpus_add_null_doc) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); cbm_sem_corpus_add_doc(c, NULL, 0); cbm_sem_corpus_add_doc(c, NULL, -1); ASSERT_EQ(cbm_sem_corpus_doc_count(c), 0); cbm_sem_corpus_free(c); PASS(); } TEST(sem_corpus_free_null) { cbm_sem_corpus_free(NULL); /* should not crash */ PASS(); } /* ── Config ──────────────────────────────────────────────────────── */ TEST(sem_get_config_defaults) { cbm_sem_config_t cfg = cbm_sem_get_config(); ASSERT_TRUE(cfg.w_tfidf > 0.0f); ASSERT_TRUE(cfg.w_ri > 0.0f); ASSERT_TRUE(cfg.threshold > 0.0f); ASSERT_TRUE(cfg.max_edges > 0); PASS(); } /* ── RaBitQ estimator quality (from-paper 4-bit quantization) ────── */ /* Deterministic pseudo-random unit vectors; validates that the quantized * inner-product estimator tracks the exact float IP within tight bounds. * These bounds gate the semantic pass's use of the codes: cosine scores are * thresholded at ~0.75, so the estimator error must be well under the * decision margin for typical pairs. */ TEST(sem_rotsq_ip_error_bounds) { enum { N = 64 }; static float vecs[N][CBM_RSQ_IN_DIM]; static cbm_rsq_code_t codes[N]; uint32_t state = 0xC0FFEEu; for (int i = 0; i < N; i++) { double norm = 0.0; for (int d = 0; d < CBM_RSQ_IN_DIM; d++) { /* xorshift32 → roughly uniform in [-1, 1] */ state ^= state << 13; state ^= state >> 17; state ^= state << 5; vecs[i][d] = ((float)(state & 0xFFFFFF) / (float)0x7FFFFF) - 1.0F; norm += (double)vecs[i][d] * (double)vecs[i][d]; } float inv = norm > 0.0 ? (float)(1.0 / sqrt(norm)) : 0.0F; for (int d = 0; d < CBM_RSQ_IN_DIM; d++) { vecs[i][d] *= inv; } cbm_rsq_encode(vecs[i], &codes[i]); } double max_err = 0.0; double sum_err = 0.0; int pairs = 0; for (int i = 0; i < N; i++) { for (int j = i; j < N; j++) { double exact = 0.0; for (int d = 0; d < CBM_RSQ_IN_DIM; d++) { exact += (double)vecs[i][d] * (double)vecs[j][d]; } double est = (double)cbm_rsq_ip(&codes[i], &codes[j]); double err = fabs(est - exact); sum_err += err; if (err > max_err) { max_err = err; } pairs++; } } double mean_err = sum_err / pairs; /* Self-IP of a unit vector must estimate ~1. */ double self_est = (double)cbm_rsq_ip(&codes[0], &codes[0]); ASSERT_TRUE(fabs(self_est - 1.0) < 0.05); /* 4-bit RaBitQ-style SQ after rotation: expect mean error well under 1% * of the unit scale and max under ~4% — comfortably inside the semantic * threshold's decision margin. */ ASSERT_TRUE(mean_err < 0.01); ASSERT_TRUE(max_err < 0.04); PASS(); } /* ── Suite ───────────────────────────────────────────────────────── */ SUITE(semantic) { RUN_TEST(sem_rotsq_ip_error_bounds); RUN_TEST(sem_tokenize_camel); RUN_TEST(sem_tokenize_snake); RUN_TEST(sem_tokenize_dot); RUN_TEST(sem_tokenize_null); RUN_TEST(sem_tokenize_max_out); RUN_TEST(sem_tokenize_abbrev_expansion); RUN_TEST(sem_cosine_identical); RUN_TEST(sem_cosine_orthogonal); RUN_TEST(sem_cosine_zero_vector); RUN_TEST(sem_cosine_negative); RUN_TEST(sem_cosine_null); RUN_TEST(sem_normalize_unit); RUN_TEST(sem_normalize_scales); RUN_TEST(sem_normalize_zero); RUN_TEST(sem_normalize_null); RUN_TEST(sem_vec_add_scaled_basic); RUN_TEST(sem_vec_add_scaled_null); RUN_TEST(sem_random_index_deterministic); RUN_TEST(sem_random_index_different_tokens); RUN_TEST(sem_random_index_null); RUN_TEST(sem_proximity_same_file); RUN_TEST(sem_proximity_same_dir); RUN_TEST(sem_proximity_different_paths); RUN_TEST(sem_proximity_null); RUN_TEST(sem_diffuse_zero_neighbors); RUN_TEST(sem_diffuse_single_neighbor); RUN_TEST(sem_corpus_new_free); RUN_TEST(sem_corpus_add_one_doc); RUN_TEST(sem_corpus_idf); RUN_TEST(sem_corpus_add_null_doc); RUN_TEST(sem_corpus_free_null); RUN_TEST(sem_get_config_defaults); }