/* * semantic.c — Algorithmic code embeddings: TF-IDF, Random Indexing, * API/Type/Decorator signatures, combined scoring, graph diffusion. * * All signals computed from graph buffer metadata — no source file reads. * Uses xxHash for deterministic random vectors. Pure C, zero dependencies. */ #include "semantic/semantic.h" #include "foundation/constants.h" #include "foundation/hash_table.h" #include "foundation/log.h" #include "foundation/profile.h" #include "foundation/platform.h" #include "foundation/compat_thread.h" #include "pipeline/worker_pool.h" #include "simhash/minhash.h" #include "nomic/code_vectors.h" #define XXH_INLINE_ALL #include "xxhash/xxhash.h" #include #include #include #include #include #include #include #include /* ── Constants ───────────────────────────────────────────────────── */ enum { TOKEN_BUF_LEN = 128, CORPUS_INIT_CAP = 4096, DOC_TOKENS_INIT = 64, RI_SEED_BASE = 0x52494E44, /* "RIND" */ }; /* Default signal weights for cbm_sem_combined_score. Must sum to ~1.0; * proximity is a multiplier applied on top. */ #define CBM_SEM_W_TFIDF 0.20F #define CBM_SEM_W_RI 0.25F #define CBM_SEM_W_MINHASH 0.10F #define CBM_SEM_W_API 0.15F #define CBM_SEM_W_TYPE 0.10F #define CBM_SEM_W_DECORATOR 0.05F #define CBM_SEM_W_STRUCT_PROFILE 0.10F #define CBM_SEM_W_DATAFLOW 0.05F /* Threshold bounds for CBM_SEMANTIC_THRESHOLD env override. */ #define CBM_SEM_THRESHOLD_MIN 0.0F #define CBM_SEM_THRESHOLD_MAX 1.0F /* Epsilon for denominator guards in cosine math. */ #define CBM_SEM_DENOM_EPS 1e-10F /* Int8 quantization bounds for vector storage (maps [-1.0, 1.0] to [-127, 127]). */ #define CBM_SEM_INT8_MAX 127.0F #define CBM_SEM_INT8_MIN (-127.0F) /* Unit vector bounds (normalized similarity / cosine value limits). */ #define CBM_SEM_UNIT_POS 1.0F #define CBM_SEM_UNIT_NEG (-1.0F) /* Proximity boost: same-file gets 1.0 + CBM_SEM_PROX_MAX_BOOST, distant gets 1.0. */ #define CBM_SEM_PROX_MAX_BOOST 0.10F /* Reflective Random Indexing blend factors for pass2 (context mixing). */ #define CBM_SEM_RRI_ALPHA 0.3F #define CBM_SEM_RRI_BETA 0.7F /* IDF smoothing constant (log base): log2(1 + docs / doc_freq). */ #define CBM_SEM_IDF_BASE 0.5F /* Strings of exactly two parts / dimensions used for co-occurrence window math. */ #define CBM_SEM_COOCCUR_STRIDE 2 /* Worker tile/chunk sizes for parallel cooccurrence passes. */ enum { CBM_SEM_WORKER_STACK_CAP = 256, CBM_SEM_TILE_SIZE = 40, CBM_SEM_SEEN_INIT_CAP = 256, CBM_SEM_RESOLVE_CHUNK = 64, CBM_SEM_COOCCUR_CHUNK = 32, CBM_SEM_RRI_TILE = 128, CBM_SEM_INT8_I_LO = -128, CBM_SEM_INT8_I_HI = 127, CBM_SEM_ATOMIC_INC = 1, CBM_SEM_RI_NONZERO_COUNT = 8, }; /* Mutex-init state machine for the lazy-initialized pretrained token map. * State 0 = not started, 1 = initializing, 2 = initialized. */ enum { MTX_STATE_UNINIT = 0, MTX_STATE_INITIALIZING = 1, MTX_STATE_READY = 2, }; /* Pretrained map ready flag (signals a one-shot atomic transition 0 → 1). */ enum { MAP_READY = 1 }; /* Numeric conversion radix for strtol (base 10 decimal). */ enum { BASE_DECIMAL = 10 }; /* ── Configuration ───────────────────────────────────────────────── */ cbm_sem_config_t cbm_sem_get_config(void) { cbm_sem_config_t cfg = { .w_tfidf = CBM_SEM_W_TFIDF, .w_ri = CBM_SEM_W_RI, .w_minhash = CBM_SEM_W_MINHASH, .w_api = CBM_SEM_W_API, .w_type = CBM_SEM_W_TYPE, .w_decorator = CBM_SEM_W_DECORATOR, .w_struct_profile = CBM_SEM_W_STRUCT_PROFILE, .w_dataflow = CBM_SEM_W_DATAFLOW, .threshold = (float)CBM_SEM_EDGE_THRESHOLD, .max_edges = CBM_SEM_MAX_EDGES, }; const char *thresh = getenv("CBM_SEMANTIC_THRESHOLD"); if (thresh) { /* strtod reports errors via endptr; reject non-numeric input silently. */ char *end = NULL; double parsed = strtod(thresh, &end); if (end != thresh && parsed > CBM_SEM_THRESHOLD_MIN && parsed <= CBM_SEM_THRESHOLD_MAX) { cfg.threshold = (float)parsed; } } return cfg; } bool cbm_sem_is_enabled(void) { const char *val = getenv("CBM_SEMANTIC_ENABLED"); return val && val[0] == '1'; } /* ── Token extraction ────────────────────────────────────────────── */ /* True for characters that terminate a token regardless of case. */ static bool is_token_delim(char c) { return c == '.' || c == '/' || c == '_' || c == '-' || c == ' ' || c == '(' || c == ')' || c == ',' || c == ':'; } /* True for a camelCase transition: uppercase letter preceded by a lowercase. */ static bool is_camel_break(const char *name, int i) { if (i <= 0) { return false; } char c = name[i]; char p = name[i - SKIP_ONE]; return c >= 'A' && c <= 'Z' && p >= 'a' && p <= 'z'; } /* Flush the current buffer as a token into out[]. */ static void flush_token(char *buf, int *blen, char **out, int *count, int max_out) { if (*blen > 0 && *count < max_out) { buf[*blen] = '\0'; out[(*count)++] = strdup(buf); } *blen = 0; } int cbm_sem_tokenize(const char *name, char **out, int max_out) { if (!name || !out || max_out <= 0) { return 0; } int count = 0; char buf[TOKEN_BUF_LEN]; int blen = 0; for (int i = 0; name[i] && count < max_out; i++) { char c = name[i]; bool split = is_token_delim(c); bool camel = is_camel_break(name, i); if (split || camel) { flush_token(buf, &blen, out, &count, max_out); if (split) { continue; } } if (blen < TOKEN_BUF_LEN - SKIP_ONE && isalnum((unsigned char)c)) { buf[blen++] = (char)tolower((unsigned char)c); } } flush_token(buf, &blen, out, &count, max_out); /* Abbreviation expansion: add expanded forms for common code abbreviations. * "err" → also add "error", "ctx" → "context", etc. */ /* Cross-language abbreviation table — covers Go, Python, JS/TS, Rust, * Java, C/C++, Ruby, PHP, Kotlin, Swift, Scala, C#, and common patterns. */ static const struct { const char *abbrev; const char *expanded; } abbrevs[] = { /* Error/exception handling */ {"err", "error"}, {"exc", "exception"}, {"ex", "exception"}, /* Context/config */ {"ctx", "context"}, {"cfg", "config"}, {"conf", "configuration"}, {"env", "environment"}, {"opt", "option"}, {"opts", "options"}, /* Request/response (HTTP, RPC) */ {"req", "request"}, {"res", "response"}, {"resp", "response"}, {"rsp", "response"}, {"hdr", "header"}, {"hdrs", "headers"}, /* Strings/formatting */ {"str", "string"}, {"fmt", "format"}, {"msg", "message"}, {"txt", "text"}, {"lbl", "label"}, {"desc", "description"}, /* Data structures */ {"buf", "buffer"}, {"arr", "array"}, {"vec", "vector"}, {"lst", "list"}, {"dict", "dictionary"}, {"tbl", "table"}, {"stk", "stack"}, {"que", "queue"}, /* Functions/callbacks */ {"fn", "function"}, {"func", "function"}, {"cb", "callback"}, {"proc", "procedure"}, {"ctor", "constructor"}, {"dtor", "destructor"}, /* Database/storage */ {"db", "database"}, {"col", "column"}, {"tbl", "table"}, {"stmt", "statement"}, {"txn", "transaction"}, {"trx", "transaction"}, {"repo", "repository"}, /* Auth/security */ {"auth", "authentication"}, {"authz", "authorization"}, {"perm", "permission"}, {"cred", "credential"}, {"tok", "token"}, {"pwd", "password"}, /* Values/types */ {"val", "value"}, {"num", "number"}, {"int", "integer"}, {"bool", "boolean"}, {"flt", "float"}, {"dbl", "double"}, /* Indexing/iteration */ {"idx", "index"}, {"iter", "iterator"}, {"elem", "element"}, {"cnt", "count"}, {"len", "length"}, {"sz", "size"}, {"pos", "position"}, {"off", "offset"}, {"cap", "capacity"}, /* Lifecycle */ {"init", "initialize"}, {"deinit", "deinitialize"}, {"alloc", "allocate"}, {"dealloc", "deallocate"}, {"del", "delete"}, {"rm", "remove"}, /* Implementation/interface */ {"impl", "implementation"}, {"iface", "interface"}, {"abs", "abstract"}, {"decl", "declaration"}, /* Parameters/attributes */ {"param", "parameter"}, {"arg", "argument"}, {"attr", "attribute"}, {"prop", "property"}, {"ret", "return"}, /* Source/destination */ {"src", "source"}, {"dst", "destination"}, {"tgt", "target"}, {"orig", "original"}, {"prev", "previous"}, {"cur", "current"}, {"tmp", "temporary"}, {"temp", "temporary"}, /* Networking/IO */ {"conn", "connection"}, {"sess", "session"}, {"sock", "socket"}, {"addr", "address"}, {"url", "uniform"}, {"srv", "server"}, {"cli", "client"}, {"svc", "service"}, {"ep", "endpoint"}, /* Management */ {"mgr", "manager"}, {"ctrl", "controller"}, {"hdlr", "handler"}, {"sched", "scheduler"}, {"disp", "dispatcher"}, {"reg", "registry"}, /* Async/concurrent */ {"chan", "channel"}, {"sem", "semaphore"}, {"mtx", "mutex"}, {"wg", "waitgroup"}, {"sig", "signal"}, {"evt", "event"}, {"sub", "subscriber"}, {"pub", "publisher"}, /* Testing */ {"spec", "specification"}, {"mock", "mock"}, {"stub", "stub"}, {"assert", "assertion"}, /* Logging/monitoring */ {"log", "logging"}, {"lvl", "level"}, {"dbg", "debug"}, {"wrn", "warning"}, {"inf", "info"}, /* Time */ {"ts", "timestamp"}, {"dur", "duration"}, {"ttl", "timetolive"}, /* Miscellaneous */ {"ver", "version"}, {"ns", "namespace"}, {"pkg", "package"}, {"mod", "module"}, {"lib", "library"}, {"dep", "dependency"}, {"ref", "reference"}, {"ptr", "pointer"}, {"obj", "object"}, {"doc", "document"}, {"cmd", "command"}, {"ops", "operations"}, {"util", "utility"}, {"hlp", "helper"}, {"ext", "extension"}, {NULL, NULL}, }; int orig_count = count; for (int t = 0; t < orig_count && count < max_out; t++) { for (int a = 0; abbrevs[a].abbrev; a++) { if (strcmp(out[t], abbrevs[a].abbrev) == 0) { out[count++] = strdup(abbrevs[a].expanded); break; } } } return count; } /* ── Dense vector operations ─────────────────────────────────────── */ float cbm_sem_cosine(const cbm_sem_vec_t *a, const cbm_sem_vec_t *b) { if (!a || !b) { return 0.0F; } float dot = 0.0F; float mag_a = 0.0F; float mag_b = 0.0F; for (int i = 0; i < CBM_SEM_DIM; i++) { dot += a->v[i] * b->v[i]; mag_a += a->v[i] * a->v[i]; mag_b += b->v[i] * b->v[i]; } float denom = sqrtf(mag_a) * sqrtf(mag_b); if (denom < CBM_SEM_DENOM_EPS) { return 0.0F; } return dot / denom; } /* Pretrained token lookup table — built lazily on first use. */ static CBMHashTable *g_pretrained_map = NULL; static _Atomic int g_pretrained_ready = 0; static cbm_mutex_t g_pretrained_mtx; static _Atomic int g_pretrained_mtx_init = 0; /* Thread-safe lazy init of the pretrained token lookup map. * Uses double-checked locking: fast path reads an atomic flag. */ static void ensure_pretrained_map(void) { if (atomic_load_explicit(&g_pretrained_ready, memory_order_acquire)) { return; } /* First-time init of the mutex itself (also needs to be thread-safe) */ int expected = MTX_STATE_UNINIT; if (atomic_compare_exchange_strong_explicit(&g_pretrained_mtx_init, &expected, MTX_STATE_INITIALIZING, memory_order_acq_rel, memory_order_acquire)) { cbm_mutex_init(&g_pretrained_mtx); atomic_store_explicit(&g_pretrained_mtx_init, MTX_STATE_READY, memory_order_release); } else { /* Spin until another thread finishes initializing the mutex */ while (atomic_load_explicit(&g_pretrained_mtx_init, memory_order_acquire) != MTX_STATE_READY) { /* brief spin */ } } cbm_mutex_lock(&g_pretrained_mtx); if (!atomic_load_explicit(&g_pretrained_ready, memory_order_acquire)) { g_pretrained_map = cbm_ht_create(PRETRAINED_TOKEN_COUNT); char idx_buf[CBM_SZ_16]; for (int i = 0; i < PRETRAINED_TOKEN_COUNT; i++) { const char *tok = PRETRAINED_TOKENS[i]; if (tok && tok[0]) { snprintf(idx_buf, sizeof(idx_buf), "%d", i); cbm_ht_set(g_pretrained_map, strdup(tok), strdup(idx_buf)); } } atomic_store_explicit(&g_pretrained_ready, MAP_READY, memory_order_release); } cbm_mutex_unlock(&g_pretrained_mtx); } void cbm_sem_ensure_ready(void) { ensure_pretrained_map(); } void cbm_sem_random_index(const char *token, cbm_sem_vec_t *out) { memset(out, 0, sizeof(*out)); if (!token) { return; } /* Try pretrained nomic-embed-code vector first (768d, distilled from 7B). */ ensure_pretrained_map(); const char *idx_str = cbm_ht_get(g_pretrained_map, token); if (idx_str) { char *end = NULL; long idx = strtol(idx_str, &end, BASE_DECIMAL); if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) { const int8_t *pvec = pretrained_vec_at((int)idx); for (int d = 0; d < CBM_SEM_DIM && d < PRETRAINED_DIM; d++) { out->v[d] = (float)pvec[d] / CBM_SEM_INT8_MAX; } return; } } /* Fallback: sparse random vector for tokens not in pretrained vocab. */ uint64_t seed = XXH3_64bits(token, strlen(token)); for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) { uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE); int pos = (int)(h % CBM_SEM_DIM); float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS; out->v[pos] += sign; } } void cbm_sem_normalize(cbm_sem_vec_t *v) { if (!v) { return; } float mag = 0.0F; for (int i = 0; i < CBM_SEM_DIM; i++) { mag += v->v[i] * v->v[i]; } mag = sqrtf(mag); if (mag < CBM_SEM_DENOM_EPS) { return; } float inv = CBM_SEM_UNIT_POS / mag; for (int i = 0; i < CBM_SEM_DIM; i++) { v->v[i] *= inv; } } void cbm_sem_vec_add_scaled(cbm_sem_vec_t *dst, const cbm_sem_vec_t *src, float scale) { if (!dst || !src) { return; } for (int i = 0; i < CBM_SEM_DIM; i++) { dst->v[i] += scale * src->v[i]; } } /* ── Corpus (IDF + Random Indexing enrichment) ───────────────────── */ typedef struct { char *token; int doc_freq; cbm_sem_vec_t enriched_vec; /* context-enriched via co-occurrence */ } corpus_entry_t; struct cbm_sem_corpus { CBMHashTable *token_map; /* token → index into entries[] */ corpus_entry_t *entries; int entry_count; int entry_cap; int doc_count; bool finalized; /* Per-document token lists for co-occurrence pass */ int **doc_token_ids; int *doc_token_counts; int doc_cap; }; static int corpus_get_or_add(cbm_sem_corpus_t *c, const char *token) { char idx_buf[CBM_SZ_16]; const char *existing = cbm_ht_get(c->token_map, token); if (existing) { char *end = NULL; long parsed = strtol(existing, &end, BASE_DECIMAL); return (end != existing) ? (int)parsed : CBM_NOT_FOUND; } if (c->entry_count >= c->entry_cap) { int new_cap = c->entry_cap < CORPUS_INIT_CAP ? CORPUS_INIT_CAP : c->entry_cap * PAIR_LEN; corpus_entry_t *grown = realloc(c->entries, (size_t)new_cap * sizeof(corpus_entry_t)); if (!grown) { return CBM_NOT_FOUND; } c->entries = grown; c->entry_cap = new_cap; } int idx = c->entry_count++; c->entries[idx].token = strdup(token); c->entries[idx].doc_freq = 0; memset(&c->entries[idx].enriched_vec, 0, sizeof(cbm_sem_vec_t)); snprintf(idx_buf, sizeof(idx_buf), "%d", idx); cbm_ht_set(c->token_map, strdup(token), strdup(idx_buf)); return idx; } cbm_sem_corpus_t *cbm_sem_corpus_new(void) { cbm_sem_corpus_t *c = calloc(SKIP_ONE, sizeof(cbm_sem_corpus_t)); if (c) { c->token_map = cbm_ht_create(CORPUS_INIT_CAP); } return c; } void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int count) { if (!corpus || !tokens || count <= 0) { return; } /* Track document for co-occurrence pass */ if (corpus->doc_count >= corpus->doc_cap) { int new_cap = corpus->doc_cap < DOC_TOKENS_INIT ? DOC_TOKENS_INIT : corpus->doc_cap * PAIR_LEN; int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); if (!grown_ids || !grown_counts) { free(grown_ids); free(grown_counts); return; } corpus->doc_token_ids = grown_ids; corpus->doc_token_counts = grown_counts; corpus->doc_cap = new_cap; } int doc_idx = corpus->doc_count++; corpus->doc_token_ids[doc_idx] = malloc((size_t)count * sizeof(int)); corpus->doc_token_counts[doc_idx] = count; /* Per-doc unique set for IDF */ int *seen = calloc((size_t)corpus->entry_cap + (size_t)count + CORPUS_INIT_CAP, sizeof(int)); int seen_count = 0; for (int i = 0; i < count; i++) { int tid = corpus_get_or_add(corpus, tokens[i]); corpus->doc_token_ids[doc_idx][i] = tid; if (tid < 0) { continue; } /* Check uniqueness for IDF (simple linear scan — tokens per doc is small) */ bool is_new = true; for (int j = 0; j < seen_count; j++) { if (seen[j] == tid) { is_new = false; break; } } if (is_new) { seen[seen_count++] = tid; corpus->entries[tid].doc_freq++; } } free(seen); } /* ── Parallel corpus batch build ──────────────────────────────────── */ /* Strategy: * Phase A (SEQUENTIAL): Scan all documents once to build the global * token_map (inserts unique tokens, assigns global IDs). This is * inherently sequential (hash table mutation), but much faster than * the current per-doc add_doc because we avoid the per-doc malloc of * the `seen` array and per-doc bookkeeping. * Phase B (PARALLEL): Each worker processes a chunk of docs, translates * tokens → global IDs via read-only token_map lookups, fills * doc_token_ids[d], and accumulates doc_freq contributions via atomics. */ typedef struct { cbm_sem_corpus_t *corpus; char **all_tokens; const int *token_counts; int max_tokens; int doc_count; _Atomic int *doc_freq_atomic; /* per-entry atomic counter (entry_count long) */ _Atomic int next_idx; } batch_resolve_ctx_t; /* Resolve one document: look up each token's global ID, fill the corpus * doc_token_ids[d], and bump the per-token doc_freq counter atomically. The * caller is responsible for ensuring `seen` has capacity for `count` ints * before calling (the worker grows its per-thread scratch buffer). */ static void batch_resolve_one_doc(batch_resolve_ctx_t *bc, int doc_index, int *seen) { int count = bc->token_counts[doc_index]; if (count <= 0) { bc->corpus->doc_token_ids[doc_index] = NULL; bc->corpus->doc_token_counts[doc_index] = 0; return; } int *ids = malloc((size_t)count * sizeof(int)); bc->corpus->doc_token_ids[doc_index] = ids; bc->corpus->doc_token_counts[doc_index] = count; int seen_count = 0; char **tokens = &bc->all_tokens[(ptrdiff_t)doc_index * bc->max_tokens]; for (int i = 0; i < count; i++) { const char *idx_str = cbm_ht_get(bc->corpus->token_map, tokens[i]); int tid = CBM_NOT_FOUND; if (idx_str) { char *end = NULL; long parsed = strtol(idx_str, &end, BASE_DECIMAL); if (end != idx_str) { tid = (int)parsed; } } ids[i] = tid; if (tid < 0) { continue; } /* Unique-per-doc check for IDF */ bool is_new = true; for (int j = 0; j < seen_count; j++) { if (seen[j] == tid) { is_new = false; break; } } if (is_new) { seen[seen_count++] = tid; atomic_fetch_add_explicit(&bc->doc_freq_atomic[tid], CBM_SEM_ATOMIC_INC, memory_order_relaxed); } } } static void batch_resolve_worker(int worker_id, void *ctx_ptr) { (void)worker_id; batch_resolve_ctx_t *bc = ctx_ptr; /* Per-worker scratch for unique-per-doc tracking */ int local_seen_cap = CBM_SEM_SEEN_INIT_CAP; int *seen = malloc((size_t)local_seen_cap * sizeof(int)); if (!seen) { return; } while (true) { int start = atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_RESOLVE_CHUNK, memory_order_relaxed); if (start >= bc->doc_count) { break; } int end = start + CBM_SEM_RESOLVE_CHUNK; if (end > bc->doc_count) { end = bc->doc_count; } for (int d = start; d < end; d++) { int count = bc->token_counts[d]; if (count > local_seen_cap) { int *grown = realloc(seen, (size_t)count * sizeof(int)); if (!grown) { continue; } seen = grown; local_seen_cap = count; } batch_resolve_one_doc(bc, d, seen); } } free(seen); } void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, const int *token_counts, int doc_count, int max_tokens_per_doc) { if (!corpus || !all_tokens || !token_counts || doc_count <= 0) { return; } /* Phase A (SEQUENTIAL): Build token_map and allocate doc arrays. * Hash table mutation can't be parallelized; strdup+insert is the cost. */ if (corpus->doc_cap < corpus->doc_count + doc_count) { int new_cap = corpus->doc_count + doc_count; int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); if (!grown_ids || !grown_counts) { free(grown_ids); free(grown_counts); return; } corpus->doc_token_ids = grown_ids; corpus->doc_token_counts = grown_counts; corpus->doc_cap = new_cap; } int base_doc = corpus->doc_count; corpus->doc_count += doc_count; for (int d = 0; d < doc_count; d++) { int count = token_counts[d]; char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc]; for (int i = 0; i < count; i++) { /* Inserts token into token_map if new; we discard return here — * Phase B will re-lookup in read-only mode to get the ID. */ (void)corpus_get_or_add(corpus, tokens[i]); } } /* Phase B (PARALLEL): Resolve tokens → IDs and count doc_freq per entry. * token_map is now read-only; each worker owns its doc range (no writes * to shared state except atomic doc_freq counters). */ _Atomic int *doc_freq_atomic = calloc((size_t)corpus->entry_count, sizeof(_Atomic int)); if (!doc_freq_atomic) { /* OOM fallback: sequential path. Roll back doc_count first since * add_doc increments it itself. */ corpus->doc_count = base_doc; for (int d = 0; d < doc_count; d++) { int count = token_counts[d]; char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc]; cbm_sem_corpus_add_doc(corpus, (const char **)tokens, count); } return; } int worker_count = cbm_default_worker_count(false); batch_resolve_ctx_t bc = { .corpus = corpus, .all_tokens = all_tokens, .token_counts = token_counts, .max_tokens = max_tokens_per_doc, .doc_count = doc_count, .doc_freq_atomic = doc_freq_atomic, }; atomic_init(&bc.next_idx, 0); /* Temporarily re-base doc arrays so workers write to base_doc..base_doc+doc_count */ corpus->doc_token_ids += base_doc; corpus->doc_token_counts += base_doc; cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; cbm_parallel_for(worker_count, batch_resolve_worker, &bc, opts); corpus->doc_token_ids -= base_doc; corpus->doc_token_counts -= base_doc; /* Phase C (SEQUENTIAL reduce): atomic counters → entries[].doc_freq */ for (int i = 0; i < corpus->entry_count; i++) { corpus->entries[i].doc_freq += atomic_load_explicit(&doc_freq_atomic[i], memory_order_relaxed); } free(doc_freq_atomic); } /* ── Parallel corpus_finalize ─────────────────────────────────────── */ /* Strategy: * 1. Precompute base RI vectors into a shared array (eliminates ~333M * redundant cbm_sem_random_index calls on kernel-scale corpora). * 2. Co-occurrence passes: partition TARGET tokens across workers so each * worker writes to a disjoint range of enriched_vec (zero contention). * Each worker still scans all documents but only accumulates for targets * in its range. Inner vector add is the parallelized work. * 3. Normalize/blend loops are trivially parallel per-entry. */ /* Reverse index: for each token id, a list of (doc_id, position_in_doc) pairs. * Built once, reused for both cooccur passes. Eliminates the O(num_chunks × doc_count) * redundant outer scan in the old algorithm. */ typedef struct { int32_t doc_id; int32_t pos; } cooccur_pos_t; typedef struct { int *offsets; /* offsets[entry_count + 1], prefix sum of occurrences */ cooccur_pos_t *flat; /* flat array of positions, total = offsets[entry_count] */ } reverse_index_t; /* Tagged source vector: sparse (~30% of tokens, 8 inline nonzeros) or dense int8 * reference into PRETRAINED_VECTOR_BLOB (no copy). Dense path converts int8→float * on the fly in the hot loop. Tested with both int8 and float32 blob storage * formats — cooccur passes are memory-bandwidth-bound, so the int8 format (4x * less source traffic) is equivalent in wall time despite the conversion cost, * while saving 90 MB of binary size. */ typedef struct { uint8_t is_sparse; /* 1 = sparse path, 0 = dense int8 reference */ uint8_t nnz; /* number of nonzeros used in sparse path */ uint16_t _pad; uint16_t indices[CBM_SEM_SPARSE_NNZE]; /* 8 * 2 = 16 bytes */ float values[CBM_SEM_SPARSE_NNZE]; /* 8 * 4 = 32 bytes */ const int8_t *dense_int8; /* points into PRETRAINED_VECTOR_BLOB */ } cbm_sem_src_entry_t; /* Inline helper: initialize a target vector from a sparse/dense source. */ static inline void sem_target_init_from_src(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src) { memset(dst, 0, sizeof(*dst)); if (src->is_sparse) { for (int k = 0; k < src->nnz; k++) { dst->v[src->indices[k]] = src->values[k]; } } else { const int8_t *s = src->dense_int8; const float inv127 = CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX; for (int d = 0; d < CBM_SEM_DIM; d++) { dst->v[d] = inv127 * (float)s[d]; } } } /* Inline helper: add weighted source into target. * Sparse path: ~8 operations, ~48 bytes source memory traffic. * Dense path: 768 mul-adds with int8→float conversion, ~768 bytes traffic. */ static inline void sem_vec_add_src_scaled(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src, float scale) { if (src->is_sparse) { for (int k = 0; k < src->nnz; k++) { dst->v[src->indices[k]] += scale * src->values[k]; } } else { const int8_t *s = src->dense_int8; const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX); for (int d = 0; d < CBM_SEM_DIM; d++) { dst->v[d] += mul * (float)s[d]; } } } /* Pass 1 context: uses sparse/int8 tagged sources (most memory-efficient). */ typedef struct { corpus_entry_t *entries; const cbm_sem_src_entry_t *src_entries; /* sparse or int8-dense per token */ int **doc_token_ids; const int *doc_token_counts; const reverse_index_t *rev; int doc_count; int entry_count; _Atomic int next_chunk; int num_chunks; int chunk_size; /* Cache-blocked tiling parameters */ int tile_size; /* targets per L2-resident tile */ } cooccur_sparse_ctx_t; /* Accumulate co-occurrence context for a single target token into `target`. * Reads neighbors within ±CBM_SEM_WINDOW positions across all documents that * reference this token id via the reverse index. */ static void cooccur_sparse_one_target(cooccur_sparse_ctx_t *cc, int tid, cbm_sem_vec_t *target) { int occ_start = cc->rev->offsets[tid]; int occ_end = cc->rev->offsets[tid + SKIP_ONE]; /* Frequent-token subsampling: stride over occurrences when a token is very * common so the enriched vector's direction is preserved (it's normalized * after) while bounding work to ~CBM_SEM_MAX_OCCUR samples. See semantic.h. */ int occ_step = SKIP_ONE; if (occ_end - occ_start > CBM_SEM_MAX_OCCUR) { occ_step = (occ_end - occ_start) / CBM_SEM_MAX_OCCUR; } for (int p = occ_start; p < occ_end; p += occ_step) { int d = cc->rev->flat[p].doc_id; int i = cc->rev->flat[p].pos; int *ids = cc->doc_token_ids[d]; int len = cc->doc_token_counts[d]; for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) { if (w == 0) { continue; } int j = i + w; if (j < 0 || j >= len) { continue; } int nid = ids[j]; if (nid < 0) { continue; } float weight = CBM_SEM_UNIT_POS / (float)abs(w); sem_vec_add_src_scaled(target, &cc->src_entries[nid], weight); } } } static void cooccur_worker_sparse(int worker_id, void *ctx_ptr) { (void)worker_id; cooccur_sparse_ctx_t *cc = ctx_ptr; while (true) { int ci = atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed); if (ci >= cc->num_chunks) { break; } int chunk_start = ci * cc->chunk_size; int chunk_end = chunk_start + cc->chunk_size; if (chunk_end > cc->entry_count) { chunk_end = cc->entry_count; } /* Cache-blocked target tiling: process tile_size targets at a time so * their vectors stay resident in L2 cache during their accumulation. */ for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) { int tile_end = tile_start + cc->tile_size; if (tile_end > chunk_end) { tile_end = chunk_end; } for (int tid = tile_start; tid < tile_end; tid++) { sem_target_init_from_src(&cc->entries[tid].enriched_vec, &cc->src_entries[tid]); cooccur_sparse_one_target(cc, tid, &cc->entries[tid].enriched_vec); } } } } /* Pass 2 (RRI) context: uses int8-quantized pass1 vectors as source. * Pass1 outputs are dense float32 post-normalization, values in [-1,1]. * We quantize to int8 once (×127) to cut source memory traffic 4x. */ typedef struct { corpus_entry_t *entries; const int8_t *pass1_q; /* [entry_count × CBM_SEM_DIM] int8 quantized pass1 */ int **doc_token_ids; const int *doc_token_counts; const reverse_index_t *rev; int doc_count; int entry_count; _Atomic int next_chunk; int num_chunks; int chunk_size; int tile_size; } cooccur_int8_ctx_t; static inline void sem_vec_add_int8_scaled(cbm_sem_vec_t *dst, const int8_t *src, float scale) { const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX); for (int d = 0; d < CBM_SEM_DIM; d++) { dst->v[d] += mul * (float)src[d]; } } /* RRI pass 2 accumulator for a single target token, reading int8-quantized * pass1 vectors as the source. */ static void cooccur_int8_one_target(cooccur_int8_ctx_t *cc, int tid, cbm_sem_vec_t *target) { int occ_start = cc->rev->offsets[tid]; int occ_end = cc->rev->offsets[tid + SKIP_ONE]; /* Frequent-token subsampling (see cooccur_sparse_one_target / semantic.h). */ int occ_step = SKIP_ONE; if (occ_end - occ_start > CBM_SEM_MAX_OCCUR) { occ_step = (occ_end - occ_start) / CBM_SEM_MAX_OCCUR; } for (int p = occ_start; p < occ_end; p += occ_step) { int d = cc->rev->flat[p].doc_id; int i = cc->rev->flat[p].pos; int *ids = cc->doc_token_ids[d]; int len = cc->doc_token_counts[d]; for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) { if (w == 0) { continue; } int j = i + w; if (j < 0 || j >= len) { continue; } int nid = ids[j]; if (nid < 0) { continue; } float weight = CBM_SEM_UNIT_POS / (float)abs(w); sem_vec_add_int8_scaled(target, &cc->pass1_q[(size_t)nid * CBM_SEM_DIM], weight); } } } static void cooccur_worker_int8(int worker_id, void *ctx_ptr) { (void)worker_id; cooccur_int8_ctx_t *cc = ctx_ptr; while (true) { int ci = atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed); if (ci >= cc->num_chunks) { break; } int chunk_start = ci * cc->chunk_size; int chunk_end = chunk_start + cc->chunk_size; if (chunk_end > cc->entry_count) { chunk_end = cc->entry_count; } for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) { int tile_end = tile_start + cc->tile_size; if (tile_end > chunk_end) { tile_end = chunk_end; } for (int tid = tile_start; tid < tile_end; tid++) { /* RRI pass 2 starts from zero (no self-init) */ memset(&cc->entries[tid].enriched_vec, 0, sizeof(cbm_sem_vec_t)); cooccur_int8_one_target(cc, tid, &cc->entries[tid].enriched_vec); } } } } typedef struct { corpus_entry_t *entries; int entry_count; _Atomic int next_idx; } norm_ctx_t; static void normalize_worker(int worker_id, void *ctx_ptr) { (void)worker_id; norm_ctx_t *nc = ctx_ptr; while (true) { int start = atomic_fetch_add_explicit(&nc->next_idx, CBM_SEM_WORKER_STACK_CAP, memory_order_relaxed); if (start >= nc->entry_count) { break; } int end = start + CBM_SEM_WORKER_STACK_CAP; if (end > nc->entry_count) { end = nc->entry_count; } for (int i = start; i < end; i++) { cbm_sem_normalize(&nc->entries[i].enriched_vec); } } } typedef struct { corpus_entry_t *entries; const cbm_sem_vec_t *pass1; int entry_count; _Atomic int next_idx; } blend_ctx_t; static void blend_worker(int worker_id, void *ctx_ptr) { (void)worker_id; blend_ctx_t *bc = ctx_ptr; while (true) { int start = atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_WORKER_STACK_CAP, memory_order_relaxed); if (start >= bc->entry_count) { break; } int end = start + CBM_SEM_WORKER_STACK_CAP; if (end > bc->entry_count) { end = bc->entry_count; } for (int i = start; i < end; i++) { cbm_sem_normalize(&bc->entries[i].enriched_vec); for (int d = 0; d < CBM_SEM_DIM; d++) { bc->entries[i].enriched_vec.v[d] = (CBM_SEM_RRI_BETA * bc->pass1[i].v[d]) + (CBM_SEM_RRI_ALPHA * bc->entries[i].enriched_vec.v[d]); } } } } typedef struct { corpus_entry_t *entries; cbm_sem_src_entry_t *src_entries; int entry_count; _Atomic int next_idx; } src_build_ctx_t; /* Build one src_entry for a token: dense float32 reference if in nomic vocab, * sparse inline representation otherwise. Collisions in the sparse hash are * merged and zeros filtered so the final representation is exactly the same * mathematical vector that the old dense path produced. */ static void build_src_entry(const char *token, cbm_sem_src_entry_t *out) { memset(out, 0, sizeof(*out)); if (!token) { out->is_sparse = SKIP_ONE; out->nnz = 0; return; } /* Dense path: direct int8 pointer into pretrained blob (zero-copy). */ const char *idx_str = cbm_ht_get(g_pretrained_map, token); if (idx_str) { char *end = NULL; long idx = strtol(idx_str, &end, BASE_DECIMAL); if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) { out->is_sparse = 0; out->dense_int8 = pretrained_vec_at((int)idx); return; } } /* Sparse path: compute 8 hash positions with collision merging. */ out->is_sparse = SKIP_ONE; uint16_t tmp_idx[CBM_SEM_SPARSE_NNZE]; float tmp_val[CBM_SEM_SPARSE_NNZE]; int count = 0; uint64_t seed = XXH3_64bits(token, strlen(token)); for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) { uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE); int pos = (int)(h % CBM_SEM_DIM); float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS; /* Merge collisions */ int found = CBM_NOT_FOUND; for (int j = 0; j < count; j++) { if (tmp_idx[j] == (uint16_t)pos) { found = j; break; } } if (found >= 0) { tmp_val[found] += sign; } else { tmp_idx[count] = (uint16_t)pos; tmp_val[count] = sign; count++; } } /* Filter zeros */ int nnz = 0; for (int j = 0; j < count; j++) { if (tmp_val[j] != 0.0F) { out->indices[nnz] = tmp_idx[j]; out->values[nnz] = tmp_val[j]; nnz++; } } out->nnz = (uint8_t)nnz; } static void src_build_worker(int worker_id, void *ctx_ptr) { (void)worker_id; src_build_ctx_t *sc = ctx_ptr; while (true) { int start = atomic_fetch_add_explicit(&sc->next_idx, CBM_SEM_WORKER_STACK_CAP, memory_order_relaxed); if (start >= sc->entry_count) { break; } int end = start + CBM_SEM_WORKER_STACK_CAP; if (end > sc->entry_count) { end = sc->entry_count; } for (int i = start; i < end; i++) { build_src_entry(sc->entries[i].token, &sc->src_entries[i]); } } } /* Quantize pass1 float vectors to int8 (× 127) for use as pass2 source. * Input vectors are unit-normalized, so values are in [-1, 1] → int8 preserves * ~99% precision. 4x less memory traffic for pass2. */ typedef struct { const corpus_entry_t *entries; int8_t *pass1_q; int entry_count; _Atomic int next_idx; } pass1_quant_ctx_t; static void pass1_quantize_worker(int worker_id, void *ctx_ptr) { (void)worker_id; pass1_quant_ctx_t *qc = ctx_ptr; while (true) { int start = atomic_fetch_add_explicit(&qc->next_idx, CBM_SEM_RRI_TILE, memory_order_relaxed); if (start >= qc->entry_count) { break; } int end = start + CBM_SEM_RRI_TILE; if (end > qc->entry_count) { end = qc->entry_count; } for (int i = start; i < end; i++) { const cbm_sem_vec_t *src = &qc->entries[i].enriched_vec; int8_t *dst = &qc->pass1_q[(size_t)i * CBM_SEM_DIM]; for (int d = 0; d < CBM_SEM_DIM; d++) { float v = src->v[d] * CBM_SEM_INT8_MAX; if (v > CBM_SEM_INT8_MAX) { v = CBM_SEM_INT8_MAX; } if (v < -CBM_SEM_INT8_MAX) { v = -CBM_SEM_INT8_MAX; } dst[d] = (int8_t)(v >= 0 ? v + CBM_SEM_IDF_BASE : v - CBM_SEM_IDF_BASE); } } } } /* Build reverse index: token_id → list of (doc_id, position) pairs. * SEQUENTIAL (fast: just pointer arithmetic + flat array fill). */ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) { reverse_index_t *rev = calloc(SKIP_ONE, sizeof(reverse_index_t)); if (!rev) { return NULL; } /* Phase A: count occurrences per token */ int *counts = calloc((size_t)corpus->entry_count + SKIP_ONE, sizeof(int)); if (!counts) { free(rev); return NULL; } long total = 0; for (int d = 0; d < corpus->doc_count; d++) { int *ids = corpus->doc_token_ids[d]; int len = corpus->doc_token_counts[d]; for (int i = 0; i < len; i++) { int tid = ids[i]; if (tid >= 0 && tid < corpus->entry_count) { counts[tid]++; total++; } } } /* Phase B: exclusive prefix sum → offsets[] */ rev->offsets = malloc(((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int)); if (!rev->offsets) { free(counts); free(rev); return NULL; } int running = 0; for (int t = 0; t < corpus->entry_count; t++) { rev->offsets[t] = running; running += counts[t]; counts[t] = 0; /* reuse as per-token fill cursor */ } rev->offsets[corpus->entry_count] = running; /* Phase C: fill flat array. Ensure allocation size > 0 even for empty * corpora (avoids malloc(0) which is implementation-defined). */ size_t flat_bytes = (total > 0 ? (size_t)total : SKIP_ONE) * sizeof(cooccur_pos_t); rev->flat = malloc(flat_bytes); if (!rev->flat) { free(rev->offsets); free(counts); free(rev); return NULL; } for (int d = 0; d < corpus->doc_count; d++) { int *ids = corpus->doc_token_ids[d]; int len = corpus->doc_token_counts[d]; for (int i = 0; i < len; i++) { int tid = ids[i]; if (tid >= 0 && tid < corpus->entry_count) { int slot = rev->offsets[tid] + counts[tid]++; rev->flat[slot].doc_id = (int32_t)d; rev->flat[slot].pos = (int32_t)i; } } } free(counts); return rev; } static void free_reverse_index(reverse_index_t *rev) { if (!rev) { return; } free(rev->offsets); free(rev->flat); free(rev); } /* Bundle of parameters shared by the finalize sub-phases. */ typedef struct { cbm_sem_corpus_t *corpus; reverse_index_t *rev; cbm_sem_src_entry_t *src_entries; int worker_count; int num_chunks; int chunk_size; int tile_size; cbm_parallel_for_opts_t opts; } finalize_params_t; /* Sub-phase 1: build tagged source vectors (sparse or dense-int8) in parallel. */ static void finalize_build_sources(finalize_params_t *p) { src_build_ctx_t sc = { .entries = p->corpus->entries, .src_entries = p->src_entries, .entry_count = p->corpus->entry_count, }; atomic_init(&sc.next_idx, 0); cbm_parallel_for(p->worker_count, src_build_worker, &sc, p->opts); } /* Sub-phases 2+3: co-occurrence pass 1 + normalize. */ static void finalize_pass1(finalize_params_t *p) { cooccur_sparse_ctx_t cc = { .entries = p->corpus->entries, .src_entries = p->src_entries, .doc_token_ids = p->corpus->doc_token_ids, .doc_token_counts = p->corpus->doc_token_counts, .rev = p->rev, .doc_count = p->corpus->doc_count, .entry_count = p->corpus->entry_count, .num_chunks = p->num_chunks, .chunk_size = p->chunk_size, .tile_size = p->tile_size, }; atomic_init(&cc.next_chunk, 0); cbm_parallel_for(p->worker_count, cooccur_worker_sparse, &cc, p->opts); norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count}; atomic_init(&nc.next_idx, 0); cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts); } /* Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */ static void finalize_pass2(finalize_params_t *p) { int8_t *pass1_q = malloc((size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t)); if (pass1_q) { pass1_quant_ctx_t qc = { .entries = p->corpus->entries, .pass1_q = pass1_q, .entry_count = p->corpus->entry_count, }; atomic_init(&qc.next_idx, 0); cbm_parallel_for(p->worker_count, pass1_quantize_worker, &qc, p->opts); } cbm_sem_vec_t *pass1 = malloc((size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t)); if (pass1) { for (int i = 0; i < p->corpus->entry_count; i++) { pass1[i] = p->corpus->entries[i].enriched_vec; } } if (pass1_q) { cooccur_int8_ctx_t cc = { .entries = p->corpus->entries, .pass1_q = pass1_q, .doc_token_ids = p->corpus->doc_token_ids, .doc_token_counts = p->corpus->doc_token_counts, .rev = p->rev, .doc_count = p->corpus->doc_count, .entry_count = p->corpus->entry_count, .num_chunks = p->num_chunks, .chunk_size = p->chunk_size, .tile_size = p->tile_size, }; atomic_init(&cc.next_chunk, 0); cbm_parallel_for(p->worker_count, cooccur_worker_int8, &cc, p->opts); } if (pass1) { blend_ctx_t bc = { .entries = p->corpus->entries, .pass1 = pass1, .entry_count = p->corpus->entry_count, }; atomic_init(&bc.next_idx, 0); cbm_parallel_for(p->worker_count, blend_worker, &bc, p->opts); free(pass1); } free(pass1_q); norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count}; atomic_init(&nc.next_idx, 0); cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts); } void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) { if (!corpus || corpus->finalized) { return; } /* Eager init before parallel dispatch to avoid lazy-init races */ ensure_pretrained_map(); int worker_count = cbm_default_worker_count(false); cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; /* Finer chunks = better load balancing for skewed token distributions. */ int num_chunks = worker_count * CBM_SEM_COOCCUR_CHUNK; if (num_chunks > corpus->entry_count) { num_chunks = corpus->entry_count; } if (num_chunks < SKIP_ONE) { num_chunks = SKIP_ONE; } int chunk_size = (corpus->entry_count + num_chunks - SKIP_ONE) / num_chunks; reverse_index_t *rev = build_reverse_index(corpus); if (!rev) { corpus->finalized = true; return; } cbm_sem_src_entry_t *src_entries = calloc((size_t)corpus->entry_count, sizeof(cbm_sem_src_entry_t)); if (!src_entries) { free_reverse_index(rev); corpus->finalized = true; return; } finalize_params_t params = { .corpus = corpus, .rev = rev, .src_entries = src_entries, .worker_count = worker_count, .num_chunks = num_chunks, .chunk_size = chunk_size, .tile_size = CBM_SEM_TILE_SIZE, .opts = opts, }; finalize_build_sources(¶ms); finalize_pass1(¶ms); finalize_pass2(¶ms); free(src_entries); free_reverse_index(rev); corpus->finalized = true; } /* Parse a decimal index string via strtol, returning CBM_NOT_FOUND on parse * failure. Centralizes the atoi-replacement pattern for token_map lookups. */ static int parse_token_index(const char *idx_str) { if (!idx_str) { return CBM_NOT_FOUND; } char *end = NULL; long parsed = strtol(idx_str, &end, BASE_DECIMAL); return (end != idx_str) ? (int)parsed : CBM_NOT_FOUND; } float cbm_sem_corpus_idf(const cbm_sem_corpus_t *corpus, const char *token) { if (!corpus || !token || corpus->doc_count == 0) { return 0.0F; } int idx = parse_token_index(cbm_ht_get(corpus->token_map, token)); if (idx < 0 || idx >= corpus->entry_count) { return 0.0F; } int df = corpus->entries[idx].doc_freq; if (df <= 0) { return 0.0F; } return logf((float)corpus->doc_count / (float)df); } const cbm_sem_vec_t *cbm_sem_corpus_ri_vec(const cbm_sem_corpus_t *corpus, const char *token) { if (!corpus || !token) { return NULL; } int idx = parse_token_index(cbm_ht_get(corpus->token_map, token)); if (idx < 0 || idx >= corpus->entry_count) { return NULL; } return &corpus->entries[idx].enriched_vec; } int cbm_sem_corpus_doc_count(const cbm_sem_corpus_t *corpus) { return corpus ? corpus->doc_count : 0; } int cbm_sem_corpus_token_count(const cbm_sem_corpus_t *corpus) { return corpus ? corpus->entry_count : 0; } const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index, const cbm_sem_vec_t **out_vec, float *out_idf) { if (!corpus || index < 0 || index >= corpus->entry_count) { return NULL; } if (out_vec) { *out_vec = &corpus->entries[index].enriched_vec; } if (out_idf && corpus->doc_count > 0) { int df = corpus->entries[index].doc_freq; *out_idf = df > 0 ? logf((float)corpus->doc_count / (float)df) : 0.0F; } return corpus->entries[index].token; } static void free_ht_kv(const char *key, void *value, void *userdata) { (void)userdata; free((void *)key); free(value); } void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus) { if (!corpus) { return; } for (int i = 0; i < corpus->entry_count; i++) { free(corpus->entries[i].token); } free(corpus->entries); for (int d = 0; d < corpus->doc_count; d++) { free(corpus->doc_token_ids[d]); } free(corpus->doc_token_ids); free(corpus->doc_token_counts); if (corpus->token_map) { cbm_ht_foreach(corpus->token_map, free_ht_kv, NULL); cbm_ht_free(corpus->token_map); } free(corpus); } /* ── Combined scoring ────────────────────────────────────────────── */ float cbm_sem_proximity(const char *path_a, const char *path_b) { if (!path_a || !path_b) { return CBM_SEM_UNIT_POS; } /* Count shared directory components */ int shared = 0; int total_a = 0; int total_b = 0; const char *a = path_a; const char *b = path_b; while (*a && *b && *a == *b) { if (*a == '/') { shared++; } a++; b++; } for (const char *p = path_a; *p; p++) { if (*p == '/') { total_a++; } } for (const char *p = path_b; *p; p++) { if (*p == '/') { total_b++; } } int max_total = total_a > total_b ? total_a : total_b; if (max_total == 0) { return CBM_SEM_UNIT_POS; } float ratio = (float)shared / (float)max_total; /* Same file = 1.10, same dir = 1.05, distant = 1.00 */ return CBM_SEM_UNIT_POS + (ratio * CBM_SEM_PROX_MAX_BOOST); } /* Cosine similarity for small float arrays (AST profile). */ static float small_cosine(const float *a, const float *b, int dims) { float dot = 0.0F; float ma = 0.0F; float mb = 0.0F; for (int i = 0; i < dims; i++) { dot += a[i] * b[i]; ma += a[i] * a[i]; mb += b[i] * b[i]; } float denom = sqrtf(ma) * sqrtf(mb); return denom < CBM_SEM_DENOM_EPS ? 0.0F : dot / denom; } /* Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when * either side is empty or the magnitude product is below the epsilon guard. */ static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *b) { if (a->tfidf_len <= 0 || b->tfidf_len <= 0) { return 0.0F; } float dot = 0.0F; float ma = 0.0F; float mb = 0.0F; int ia = 0; int ib = 0; while (ia < a->tfidf_len && ib < b->tfidf_len) { if (a->tfidf_indices[ia] == b->tfidf_indices[ib]) { dot += a->tfidf_weights[ia] * b->tfidf_weights[ib]; ia++; ib++; } else if (a->tfidf_indices[ia] < b->tfidf_indices[ib]) { ia++; } else { ib++; } } for (int i = 0; i < a->tfidf_len; i++) { ma += a->tfidf_weights[i] * a->tfidf_weights[i]; } for (int i = 0; i < b->tfidf_len; i++) { mb += b->tfidf_weights[i] * b->tfidf_weights[i]; } float denom = sqrtf(ma) * sqrtf(mb); return denom > CBM_SEM_DENOM_EPS ? (dot / denom) : 0.0F; } float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, const cbm_sem_config_t *cfg) { if (!a || !b || !cfg) { return 0.0F; } /* Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold, * the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair. * Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate * (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers, * freeing the edge budget for true semantic leaps (vocabulary-bridged relations). */ if (a->has_minhash && b->has_minhash) { double early_j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, (const cbm_minhash_t *)b->minhash); if (early_j >= CBM_MINHASH_JACCARD_THRESHOLD) { return 0.0F; } } float score = cfg->w_tfidf * sparse_tfidf_cosine(a, b); /* Signal 2: Random Indexing */ score += cfg->w_ri * cbm_rsq_ip(&a->ri_code, &b->ri_code); /* Signal 3: MinHash Jaccard */ if (a->has_minhash && b->has_minhash) { double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, (const cbm_minhash_t *)b->minhash); score += cfg->w_minhash * (float)j; } /* Signal 4: API Signatures */ score += cfg->w_api * cbm_rsq_ip(&a->api_code, &b->api_code); /* Signal 5: Type Signatures */ score += cfg->w_type * cbm_rsq_ip(&a->type_code, &b->type_code); /* Signal 7: Decorator Pattern */ score += cfg->w_decorator * cbm_rsq_ip(&a->deco_code, &b->deco_code); /* Signal 8+9+11: Structural profile + data flow + Halstead */ float sp_score = small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS); score += cfg->w_struct_profile * sp_score; /* Signal 6: Module proximity (multiplier, not additive). * Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking. * Clamp the final product to [0, 1] so the output stays within valid * cosine-similarity range. Without the clamp, a 0.95 base × 1.10 proximity * would emit a 1.045 score which violates the semantic of "similarity" and * breaks downstream consumers that expect a normalized value. */ score *= cbm_sem_proximity(a->file_path, b->file_path); if (score > CBM_SEM_UNIT_POS) { score = CBM_SEM_UNIT_POS; } if (score < 0.0F) { score = 0.0F; } return score; } /* ── Graph diffusion ─────────────────────────────────────────────── */ void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, int neighbor_count, float alpha) { if (!combined || !neighbors || neighbor_count <= 0) { return; } /* Blend: combined = (1-α) × combined + α × mean(neighbors) */ cbm_sem_vec_t mean; memset(&mean, 0, sizeof(mean)); for (int n = 0; n < neighbor_count; n++) { for (int i = 0; i < CBM_SEM_DIM; i++) { mean.v[i] += neighbors[n].v[i]; } } float inv_n = CBM_SEM_UNIT_POS / (float)neighbor_count; float one_minus_alpha = CBM_SEM_UNIT_POS - alpha; for (int i = 0; i < CBM_SEM_DIM; i++) { combined->v[i] = (one_minus_alpha * combined->v[i]) + (alpha * mean.v[i] * inv_n); } cbm_sem_normalize(combined); }