chore: import upstream snapshot with attribution

src/semantic/ast_profile.c

@@ -0,0 +1,417 @@
+/*
+ * ast_profile.c — Extraction-time AST structural signals.
+ *
+ * Walks a function body AST (alongside MinHash) to compute:
+ *   Signal 8:  Control flow counts, nesting depth, expression types
+ *   Signal 9:  Approximate data flow (params in returns/conditions)
+ *   Signal 11: Halstead-lite (unique/total operators/operands)
+ *
+ * Pure functions — thread-safe, no shared state.
+ */
+#include "semantic/ast_profile.h"
+#include "foundation/constants.h"
+#include "tree_sitter/api.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+/* ── Node type classification ────────────────────────────────────── */
+
+enum {
+    WALK_STACK_CAP = 2048,
+    HALSTEAD_SET_SIZE = 512,
+    HALSTEAD_SET_MASK = 511,
+    PROFILE_FIELD_COUNT = 25,
+    DEPTH_SCALE = 10,
+    BASE_DECIMAL_AST = 10,
+    HALSTEAD_HASH_MUL = 31,
+};
+
+typedef struct {
+    TSNode node;
+    int depth;
+} profile_frame_t;
+
+static bool is_control_if(const char *k) {
+    return strcmp(k, "if_statement") == 0 || strcmp(k, "if_expression") == 0 ||
+           strcmp(k, "elif_clause") == 0;
+}
+
+static bool is_control_for(const char *k) {
+    return strcmp(k, "for_statement") == 0 || strcmp(k, "for_range_loop") == 0 ||
+           strcmp(k, "for_expression") == 0 || strcmp(k, "for_in_clause") == 0;
+}
+
+static bool is_control_while(const char *k) {
+    return strcmp(k, "while_statement") == 0 || strcmp(k, "while_expression") == 0 ||
+           strcmp(k, "do_statement") == 0;
+}
+
+static bool is_control_switch(const char *k) {
+    return strcmp(k, "switch_statement") == 0 || strcmp(k, "switch_expression") == 0 ||
+           strcmp(k, "match_expression") == 0 || strcmp(k, "type_switch_statement") == 0;
+}
+
+static bool is_control_try(const char *k) {
+    return strcmp(k, "try_statement") == 0 || strcmp(k, "try_expression") == 0 ||
+           strcmp(k, "catch_clause") == 0 || strcmp(k, "except_clause") == 0;
+}
+
+static bool is_return(const char *k) {
+    return strcmp(k, "return_statement") == 0 || strcmp(k, "return_expression") == 0;
+}
+
+static bool is_comparison(const char *k) {
+    return strcmp(k, "binary_expression") == 0 || strcmp(k, "comparison_operator") == 0 ||
+           strcmp(k, "boolean_operator") == 0;
+}
+
+static bool is_arithmetic(const char *k) {
+    return strcmp(k, "unary_expression") == 0 || strcmp(k, "update_expression") == 0;
+}
+
+static bool is_assignment(const char *k) {
+    return strcmp(k, "assignment_expression") == 0 || strcmp(k, "assignment_statement") == 0 ||
+           strcmp(k, "augmented_assignment") == 0 || strcmp(k, "short_var_declaration") == 0;
+}
+
+static bool is_string_lit(const char *k) {
+    return strcmp(k, "string") == 0 || strcmp(k, "string_literal") == 0 ||
+           strcmp(k, "interpreted_string_literal") == 0 || strcmp(k, "raw_string_literal") == 0;
+}
+
+static bool is_number_lit(const char *k) {
+    return strcmp(k, "number") == 0 || strcmp(k, "integer") == 0 || strcmp(k, "float") == 0 ||
+           strcmp(k, "integer_literal") == 0 || strcmp(k, "float_literal") == 0;
+}
+
+static bool is_bool_lit(const char *k) {
+    return strcmp(k, "true") == 0 || strcmp(k, "false") == 0;
+}
+
+static bool is_operator_node(const char *k) {
+    /* Named nodes that represent operations (not data). */
+    return is_control_if(k) || is_control_for(k) || is_control_while(k) || is_control_switch(k) ||
+           is_control_try(k) || is_return(k) || is_comparison(k) || is_arithmetic(k) ||
+           is_assignment(k) || strcmp(k, "call_expression") == 0 ||
+           strcmp(k, "member_expression") == 0 || strcmp(k, "subscript_expression") == 0;
+}
+
+static bool is_identifier(const char *k) {
+    return strcmp(k, "identifier") == 0 || strcmp(k, "field_identifier") == 0 ||
+           strcmp(k, "property_identifier") == 0 || strcmp(k, "type_identifier") == 0;
+}
+
+/* Simple hash set for Halstead unique counting (open addressing). */
+static bool halstead_insert(uint32_t *set, const char *key) {
+    uint32_t h = 0;
+    for (const char *p = key; *p; p++) {
+        h = (h * HALSTEAD_HASH_MUL) + (uint32_t)*p;
+    }
+    uint32_t idx = h & HALSTEAD_SET_MASK;
+    for (int probe = 0; probe < HALSTEAD_SET_SIZE; probe++) {
+        uint32_t slot = (idx + (uint32_t)probe) & HALSTEAD_SET_MASK;
+        if (set[slot] == 0) {
+            set[slot] = h | SKIP_ONE;
+            return true; /* new */
+        }
+        if (set[slot] == (h | SKIP_ONE)) {
+            return false; /* existing */
+        }
+    }
+    return false;
+}
+
+/* Check if an identifier matches any parameter name. */
+static bool is_param_name(const char *ident, const char *source, const char **param_names,
+                          int param_count) {
+    if (!ident || !source || !param_names || param_count == 0) {
+        return false;
+    }
+    for (int i = 0; i < param_count; i++) {
+        if (param_names[i] && strcmp(ident, param_names[i]) == 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/* ── Main computation ────────────────────────────────────────────── */
+
+/* Count control-flow statement kinds (if/for/while/switch/try/return). */
+static void accumulate_control_flow(const char *kind, cbm_ast_profile_t *out, bool *in_return) {
+    if (is_control_if(kind)) {
+        out->if_count++;
+    }
+    if (is_control_for(kind)) {
+        out->for_count++;
+    }
+    if (is_control_while(kind)) {
+        out->while_count++;
+    }
+    if (is_control_switch(kind)) {
+        out->switch_count++;
+    }
+    if (is_control_try(kind)) {
+        out->try_count++;
+    }
+    if (is_return(kind)) {
+        out->return_count++;
+        *in_return = true;
+    }
+}
+
+/* Count expression- and literal-class counters for one node. */
+static void accumulate_expressions(const char *kind, cbm_ast_profile_t *out) {
+    if (is_comparison(kind)) {
+        out->comparison_ops++;
+    }
+    if (is_arithmetic(kind)) {
+        out->arithmetic_ops++;
+    }
+    if (strcmp(kind, "not_operator") == 0 || strcmp(kind, "boolean_operator") == 0) {
+        out->logical_ops++;
+    }
+    if (is_assignment(kind)) {
+        out->assignment_count++;
+        out->variable_reassigns++;
+    }
+    if (is_string_lit(kind)) {
+        out->string_literals++;
+    }
+    if (is_number_lit(kind)) {
+        out->number_literals++;
+    }
+    if (is_bool_lit(kind)) {
+        out->bool_literals++;
+    }
+}
+
+/* Accumulate Halstead operator/operand counts.  Leaf identifiers, literals,
+ * and booleans are "operands"; recognized statement/expression kinds are
+ * "operators".  Uses an open-addressed hash set for unique counting. */
+static void accumulate_halstead(const char *kind, uint32_t child_count, uint32_t *op_set,
+                                uint32_t *operand_set, cbm_ast_profile_t *out) {
+    if (is_operator_node(kind)) {
+        out->total_operators++;
+        if (halstead_insert(op_set, kind)) {
+            out->unique_operators++;
+        }
+    }
+    if (child_count == 0 &&
+        (is_identifier(kind) || is_string_lit(kind) || is_number_lit(kind) || is_bool_lit(kind))) {
+        out->total_operands++;
+        if (halstead_insert(operand_set, kind)) {
+            out->unique_operands++;
+        }
+        out->body_tokens++;
+    }
+}
+
+/* Data-flow counter update: bump params_in_returns / params_in_conditions
+ * when the current leaf identifier names a parameter and we're inside the
+ * corresponding syntactic scope. */
+static void accumulate_data_flow(TSNode node, const char *kind, uint32_t child_count,
+                                 const char *source, const char **param_names, int param_count,
+                                 bool in_return, bool in_condition, cbm_ast_profile_t *out) {
+    if (!(child_count == 0 && is_identifier(kind) && source)) {
+        return;
+    }
+    uint32_t start = ts_node_start_byte(node);
+    uint32_t end = ts_node_end_byte(node);
+    if (end <= start || (end - start) >= CBM_SZ_128) {
+        return;
+    }
+    char ident_buf[CBM_SZ_128];
+    int ilen = (int)(end - start);
+    memcpy(ident_buf, source + start, (size_t)ilen);
+    ident_buf[ilen] = '\0';
+    if (!is_param_name(ident_buf, source, param_names, param_count)) {
+        return;
+    }
+    if (in_return) {
+        out->params_in_returns++;
+    }
+    if (in_condition) {
+        out->params_in_conditions++;
+    }
+}
+
+bool cbm_ast_profile_compute(TSNode func_body, const char *source, const char **param_names,
+                             int param_count, cbm_ast_profile_t *out) {
+    if (ts_node_is_null(func_body)) {
+        return false;
+    }
+
+    memset(out, 0, sizeof(*out));
+    out->param_count = (uint16_t)param_count;
+
+    uint32_t op_set[HALSTEAD_SET_SIZE];
+    uint32_t operand_set[HALSTEAD_SET_SIZE];
+    memset(op_set, 0, sizeof(op_set));
+    memset(operand_set, 0, sizeof(operand_set));
+
+    int total_depth = 0;
+    int node_count = 0;
+    bool in_return = false;
+    bool in_condition = false;
+
+    profile_frame_t stack[WALK_STACK_CAP];
+    int top = 0;
+    stack[top++] = (profile_frame_t){func_body, 0};
+
+    while (top > 0) {
+        profile_frame_t frame = stack[--top];
+        TSNode node = frame.node;
+        int depth = frame.depth;
+        uint32_t child_count = ts_node_child_count(node);
+        const char *kind = ts_node_type(node);
+
+        if (!ts_node_is_named(node) && child_count == 0) {
+            /* Anonymous leaf (punctuation, keywords) — skip. */
+            goto push_children;
+        }
+
+        node_count++;
+        total_depth += depth;
+
+        if ((uint16_t)depth > out->max_nesting_depth) {
+            out->max_nesting_depth = (uint16_t)depth;
+        }
+
+        accumulate_control_flow(kind, out, &in_return);
+        accumulate_expressions(kind, out);
+        accumulate_halstead(kind, child_count, op_set, operand_set, out);
+        accumulate_data_flow(node, kind, child_count, source, param_names, param_count, in_return,
+                             in_condition, out);
+
+        /* Track context for data flow: are we inside a condition? */
+        if (is_control_if(kind) || is_control_while(kind)) {
+            in_condition = true;
+        }
+
+    push_children:
+        /* Reset context flags when leaving return/condition scope */
+        if (is_return(kind)) {
+            in_return = false;
+        }
+        if (child_count > 0 && (is_control_if(kind) || is_control_while(kind))) {
+            in_condition = false;
+        }
+
+        /* Push children in reverse order */
+        for (int i = (int)child_count - SKIP_ONE; i >= 0 && top < WALK_STACK_CAP; i--) {
+            stack[top++] = (profile_frame_t){ts_node_child(node, (uint32_t)i), depth + SKIP_ONE};
+        }
+    }
+
+    /* Compute averages */
+    if (node_count > 0) {
+        out->avg_nesting_depth_x10 = (uint16_t)((total_depth * DEPTH_SCALE) / node_count);
+    }
+
+    return node_count > 0;
+}
+
+/* ── Serialization ───────────────────────────────────────────────── */
+
+void cbm_ast_profile_to_str(const cbm_ast_profile_t *p, char *buf, int bufsize) {
+    if (!p || !buf || bufsize < SKIP_ONE) {
+        if (buf && bufsize > 0) {
+            buf[0] = '\0';
+        }
+        return;
+    }
+    snprintf(buf, (size_t)bufsize,
+             "%u,%u,%u,%u,%u,%u,%u,%u,"
+             "%u,%u,%u,%u,"
+             "%u,%u,%u,"
+             "%u,%u,%u,%u,"
+             "%u,%u,%u,%u,"
+             "%u,%u",
+             p->if_count, p->for_count, p->while_count, p->switch_count, p->try_count,
+             p->return_count, p->max_nesting_depth, p->avg_nesting_depth_x10, p->comparison_ops,
+             p->arithmetic_ops, p->logical_ops, p->assignment_count, p->string_literals,
+             p->number_literals, p->bool_literals, p->param_count, p->params_in_returns,
+             p->params_in_conditions, p->variable_reassigns, p->unique_operators,
+             p->unique_operands, p->total_operators, p->total_operands, p->body_lines,
+             p->body_tokens);
+}
+
+bool cbm_ast_profile_from_str(const char *str, cbm_ast_profile_t *out) {
+    if (!str || !out) {
+        return false;
+    }
+    memset(out, 0, sizeof(*out));
+    /* Field layout must match cbm_ast_profile_to_str() exactly. */
+    uint16_t *fields[PROFILE_FIELD_COUNT] = {
+        &out->if_count,           &out->for_count,
+        &out->while_count,        &out->switch_count,
+        &out->try_count,          &out->return_count,
+        &out->max_nesting_depth,  &out->avg_nesting_depth_x10,
+        &out->comparison_ops,     &out->arithmetic_ops,
+        &out->logical_ops,        &out->assignment_count,
+        &out->string_literals,    &out->number_literals,
+        &out->bool_literals,      &out->param_count,
+        &out->params_in_returns,  &out->params_in_conditions,
+        &out->variable_reassigns, &out->unique_operators,
+        &out->unique_operands,    &out->total_operators,
+        &out->total_operands,     &out->body_lines,
+        &out->body_tokens,
+    };
+    const char *p = str;
+    for (int i = 0; i < PROFILE_FIELD_COUNT; i++) {
+        char *end = NULL;
+        unsigned long parsed = strtoul(p, &end, BASE_DECIMAL_AST);
+        if (end == p) {
+            return false;
+        }
+        *fields[i] = (uint16_t)parsed;
+        p = end;
+        /* Skip the comma separator between fields. */
+        if (i + SKIP_ONE < PROFILE_FIELD_COUNT) {
+            if (*p != ',') {
+                return false;
+            }
+            p++;
+        }
+    }
+    return true;
+}
+
+void cbm_ast_profile_to_vector(const cbm_ast_profile_t *p, float *out) {
+    if (!p || !out) {
+        return;
+    }
+    /* Normalize each field to [0,1] range using reasonable maximums. */
+    enum { MAX_COUNT = 100, MAX_DEPTH = 20, MAX_HALSTEAD = 200, MAX_TOKENS = 2000 };
+    int i = 0;
+    out[i++] = (float)p->if_count / MAX_COUNT;
+    out[i++] = (float)p->for_count / MAX_COUNT;
+    out[i++] = (float)p->while_count / MAX_COUNT;
+    out[i++] = (float)p->switch_count / MAX_COUNT;
+    out[i++] = (float)p->try_count / MAX_COUNT;
+    out[i++] = (float)p->return_count / MAX_COUNT;
+    out[i++] = (float)p->max_nesting_depth / MAX_DEPTH;
+    out[i++] = (float)p->avg_nesting_depth_x10 / (MAX_DEPTH * DEPTH_SCALE);
+    out[i++] = (float)p->comparison_ops / MAX_COUNT;
+    out[i++] = (float)p->arithmetic_ops / MAX_COUNT;
+    out[i++] = (float)p->logical_ops / MAX_COUNT;
+    out[i++] = (float)p->assignment_count / MAX_COUNT;
+    out[i++] = (float)p->string_literals / MAX_COUNT;
+    out[i++] = (float)p->number_literals / MAX_COUNT;
+    out[i++] = (float)p->bool_literals / MAX_COUNT;
+    out[i++] = (float)p->param_count / MAX_DEPTH;
+    out[i++] = (float)p->params_in_returns / MAX_COUNT;
+    out[i++] = (float)p->params_in_conditions / MAX_COUNT;
+    out[i++] = (float)p->variable_reassigns / MAX_COUNT;
+    out[i++] = (float)p->unique_operators / MAX_HALSTEAD;
+    out[i++] = (float)p->unique_operands / MAX_HALSTEAD;
+    out[i++] = (float)p->total_operators / MAX_HALSTEAD;
+    out[i++] = (float)p->total_operands / MAX_HALSTEAD;
+    out[i++] = (float)p->body_lines / MAX_TOKENS;
+    out[i++] = (float)p->body_tokens / MAX_TOKENS;
+}

src/semantic/ast_profile.h

@@ -0,0 +1,82 @@
+/*
+ * ast_profile.h — Extraction-time AST structural signals.
+ *
+ * Computed during the existing AST walk (alongside MinHash) with near-zero
+ * marginal cost.  Captures control flow shape, expression types, data flow
+ * approximations, and Halstead-lite metrics.
+ *
+ * Stored as a compact comma-separated string in properties_json ("sp" key)
+ * and decoded in the semantic post-pass for combined similarity scoring.
+ */
+#ifndef CBM_AST_PROFILE_H
+#define CBM_AST_PROFILE_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+/* Forward declare tree-sitter types to avoid pulling in api.h everywhere. */
+typedef struct TSNode TSNode;
+
+/* ── Structural profile: ~27 dimensions ──────────────────────────── */
+
+typedef struct {
+    /* Signal 8: Control flow counts */
+    uint16_t if_count;
+    uint16_t for_count;
+    uint16_t while_count;
+    uint16_t switch_count;
+    uint16_t try_count;
+    uint16_t return_count;
+    uint16_t max_nesting_depth;
+    uint16_t avg_nesting_depth_x10; /* ×10 for fixed-point */
+
+    /* Signal 8: Expression type distribution (counts) */
+    uint16_t comparison_ops;
+    uint16_t arithmetic_ops;
+    uint16_t logical_ops;
+    uint16_t assignment_count;
+
+    /* Signal 8: Literal type distribution */
+    uint16_t string_literals;
+    uint16_t number_literals;
+    uint16_t bool_literals;
+
+    /* Signal 9: Approximate data flow */
+    uint16_t param_count;
+    uint16_t params_in_returns;    /* params referenced in return statements */
+    uint16_t params_in_conditions; /* params referenced in if conditions */
+    uint16_t variable_reassigns;   /* same-name assignments */
+
+    /* Signal 11: Halstead-lite */
+    uint16_t unique_operators;
+    uint16_t unique_operands;
+    uint16_t total_operators;
+    uint16_t total_operands;
+
+    /* Body metrics */
+    uint16_t body_lines;
+    uint16_t body_tokens; /* leaf AST node count */
+} cbm_ast_profile_t;
+
+/* Number of dimensions when serialized as a float vector. */
+enum { CBM_AST_PROFILE_DIMS = 25 };
+
+/* Compute AST structural profile from a function body node.
+ * Pure function — thread-safe, no shared state.
+ * Returns true if the profile was computed (function was large enough). */
+bool cbm_ast_profile_compute(TSNode func_body, const char *source, const char **param_names,
+                             int param_count, cbm_ast_profile_t *out);
+
+/* Encode profile to a compact comma-separated string.
+ * buf must be at least CBM_AST_PROFILE_BUF bytes. */
+enum { CBM_AST_PROFILE_BUF = 200 };
+void cbm_ast_profile_to_str(const cbm_ast_profile_t *p, char *buf, int bufsize);
+
+/* Decode profile from comma-separated string. Returns true on success. */
+bool cbm_ast_profile_from_str(const char *str, cbm_ast_profile_t *out);
+
+/* Convert profile to a normalized float vector for cosine similarity.
+ * out must have CBM_AST_PROFILE_DIMS elements. */
+void cbm_ast_profile_to_vector(const cbm_ast_profile_t *p, float *out);
+
+#endif /* CBM_AST_PROFILE_H */

src/semantic/rotsq.c

@@ -0,0 +1,127 @@
+/*
+ * rotsq.c — see rotsq.h. From-paper implementation, no dependencies.
+ */
+#include "semantic/rotsq.h"
+
+#include <string.h>
+
+#define XXH_INLINE_ALL
+#include "xxhash/xxhash.h"
+
+/* ── Deterministic ±1 diagonal (seeded, generated once) ─────────────── */
+
+static float g_rsq_diag[CBM_RSQ_DIM];
+static int g_rsq_diag_ready = 0;
+
+static void rsq_init_diag(void) {
+    if (g_rsq_diag_ready) {
+        return;
+    }
+    for (int d = 0; d < CBM_RSQ_DIM; d++) {
+        uint64_t h = XXH3_64bits_withSeed(&d, sizeof(d), 0x5bd1e995u);
+        g_rsq_diag[d] = (h & 1u) ? 1.0F : -1.0F;
+    }
+    g_rsq_diag_ready = 1;
+}
+
+/* ── Fast Walsh–Hadamard Transform (in place, unnormalized) ─────────── */
+
+static void rsq_fwht(float *v) {
+    for (int len = 1; len < CBM_RSQ_DIM; len <<= 1) {
+        for (int i = 0; i < CBM_RSQ_DIM; i += len << 1) {
+            for (int j = i; j < i + len; j++) {
+                float a = v[j];
+                float b = v[j + len];
+                v[j] = a + b;
+                v[j + len] = a - b;
+            }
+        }
+    }
+}
+
+/* ── Encode ─────────────────────────────────────────────────────────── */
+
+void cbm_rsq_encode(const float *v, cbm_rsq_code_t *out) {
+    rsq_init_diag();
+
+    float rot[CBM_RSQ_DIM];
+    for (int d = 0; d < CBM_RSQ_IN_DIM; d++) {
+        rot[d] = v[d] * g_rsq_diag[d];
+    }
+    for (int d = CBM_RSQ_IN_DIM; d < CBM_RSQ_DIM; d++) {
+        rot[d] = 0.0F;
+    }
+    rsq_fwht(rot);
+    /* Normalize the transform so the rotation is orthonormal (H/√D): keeps
+     * the coordinates in a data-independent range and makes the estimated IP
+     * directly comparable to the pre-rotation IP. */
+    const float inv_sqrt_d = 1.0F / 32.0F; /* 1/√1024 */
+    float lo = rot[0] * inv_sqrt_d;
+    float hi = lo;
+    for (int d = 0; d < CBM_RSQ_DIM; d++) {
+        rot[d] *= inv_sqrt_d;
+        if (rot[d] < lo) {
+            lo = rot[d];
+        }
+        if (rot[d] > hi) {
+            hi = rot[d];
+        }
+    }
+
+    /* Per-vector scalar quantization over [lo, hi] at CBM_RSQ_BITS. */
+    float range = hi - lo;
+    float step = range > 0.0F ? range / (float)CBM_RSQ_LEVELS : 1.0F;
+    out->offset = lo;
+    out->scale = step;
+
+    int32_t sum = 0;
+    memset(out->codes, 0, sizeof(out->codes));
+    for (int d = 0; d < CBM_RSQ_DIM; d++) {
+        float q = (rot[d] - lo) / step;
+        int32_t c = (int32_t)(q + 0.5F);
+        if (c < 0) {
+            c = 0;
+        }
+        if (c > CBM_RSQ_LEVELS) {
+            c = CBM_RSQ_LEVELS;
+        }
+        sum += c;
+        if (d & 1) {
+            out->codes[d >> 1] |= (uint8_t)(c << 4);
+        } else {
+            out->codes[d >> 1] |= (uint8_t)c;
+        }
+    }
+    out->code_sum = sum;
+}
+
+/* ── Estimated inner product from two codes ─────────────────────────── */
+
+float cbm_rsq_ip(const cbm_rsq_code_t *a, const cbm_rsq_code_t *b) {
+    /* Integer dot of the 4-bit codes. */
+    int64_t dot = 0;
+    for (int i = 0; i < CBM_RSQ_CODE_BYTES; i++) {
+        uint8_t ba = a->codes[i];
+        uint8_t bb = b->codes[i];
+        dot += (int64_t)(ba & 0x0F) * (bb & 0x0F);
+        dot += (int64_t)(ba >> 4) * (bb >> 4);
+    }
+    /* x_i ≈ oa + sa·ca_i, y_i ≈ ob + sb·cb_i (in the rotated space, where the
+     * IP equals the original IP because the rotation is orthonormal). */
+    double d = (double)CBM_RSQ_DIM;
+    double ip = d * (double)a->offset * (double)b->offset +
+                (double)a->offset * (double)b->scale * (double)b->code_sum +
+                (double)b->offset * (double)a->scale * (double)a->code_sum +
+                (double)a->scale * (double)b->scale * (double)dot;
+    return (float)ip;
+}
+
+/* ── Dequantize into the rotated basis ──────────────────────────────── */
+
+void cbm_rsq_decode(const cbm_rsq_code_t *c, float *out) {
+    for (int i = 0; i < CBM_RSQ_CODE_BYTES; i++) {
+        uint8_t b = c->codes[i];
+        out[i * 2] = c->offset + c->scale * (float)(b & 0x0F);
+        out[i * 2 + 1] = c->offset + c->scale * (float)(b >> 4);
+    }
+}

src/semantic/rotsq.h

@@ -0,0 +1,67 @@
+/*
+ * rotsq.h — RaBitQ-style B-bit vector quantization (from-paper, plain C11).
+ *
+ * Implements the core of Extended RaBitQ (Gao et al., SIGMOD 2024/2025;
+ * arXiv:2405.12497, arXiv:2409.09913) without the reference library (which is
+ * C++ and bundles the Eigen linear-algebra dependency; nothing here is vendored
+ * or derived from it — this file is written from the papers). Named rotsq, not
+ * rabitq: this is the
+ * FAMILY core (randomized rotation + per-vector scalar quantization + exact
+ * code-expansion estimator), not the papers' exact codebook construction —
+ * the name should not overclaim fidelity. Rotate each vector with a deterministic
+ * randomized transform, then scalar-quantize the rotated coordinates to B
+ * bits. The rotation spreads a vector's mass evenly across coordinates
+ * (rotated coords of a unit vector are near-Gaussian), which makes plain
+ * scalar quantization behave near-optimally — that observation IS RaBitQ.
+ *
+ * Rotation: random ±1 diagonal (XXH3-seeded, reproducible — same idiom as the
+ * LSH hyperplanes) followed by a Fast Walsh–Hadamard Transform, inputs padded
+ * to the next power of two (768 → 1024). Orthogonal up to a known 1/√D scale.
+ *
+ * Inner product between two ENCODED vectors is estimated from the codes with
+ * the exact SQ expansion (deterministic — a pure function of the two codes):
+ *   x_i ≈ ox + sx·cx_i  ⇒  ⟨x,y⟩ ≈ D·ox·oy + ox·sy·Σcy + oy·sx·Σcx
+ *                                   + sx·sy·Σ(cx_i·cy_i)
+ * with Σcx/Σcy precomputed per vector, so scoring one pair is one integer
+ * dot product (u8×u8) plus four multiplies.
+ *
+ * Memory at B=4, D=1024: 512 B codes + 12 B metadata per vector, vs 3 072 B
+ * for 768 raw floats — ~6× per vector. Cosine of normalized inputs is the
+ * estimated IP directly (rotation preserves norms up to the fixed scale,
+ * which cancels in the scale factors).
+ */
+#ifndef CBM_SEMANTIC_ROTSQ_H
+#define CBM_SEMANTIC_ROTSQ_H
+
+#include <stdint.h>
+
+enum {
+    CBM_RSQ_IN_DIM = 768,                 /* input dimension (CBM_SEM_DIM) */
+    CBM_RSQ_DIM = 1024,                   /* padded pow2 rotation dimension */
+    CBM_RSQ_BITS = 4,                     /* bits per coordinate */
+    CBM_RSQ_LEVELS = 15,                  /* (1 << CBM_RSQ_BITS) - 1 */
+    CBM_RSQ_CODE_BYTES = CBM_RSQ_DIM / 2, /* two 4-bit codes per byte */
+};
+
+typedef struct {
+    uint8_t codes[CBM_RSQ_CODE_BYTES]; /* packed 4-bit codes, little nibble first */
+    float scale;                       /* per-vector dequant scale  */
+    float offset;                      /* per-vector dequant offset */
+    int32_t code_sum;                  /* Σ codes (for the IP expansion) */
+} cbm_rsq_code_t;
+
+/* Encode a CBM_RSQ_IN_DIM float vector (zero-padded to CBM_RSQ_DIM, rotated,
+ * quantized). Deterministic: the rotation is fixed at build time. */
+void cbm_rsq_encode(const float *v, cbm_rsq_code_t *out);
+
+/* Estimated inner product of the two ORIGINAL vectors from their codes.
+ * Deterministic pure function of the codes. */
+float cbm_rsq_ip(const cbm_rsq_code_t *a, const cbm_rsq_code_t *b);
+
+/* Dequantize a code into the ROTATED space (CBM_RSQ_DIM floats). Note: this
+ * is the rotated basis, not the original one — fine for basis-agnostic
+ * consumers (LSH hyperplane signs), wrong for anything expecting original
+ * coordinates. */
+void cbm_rsq_decode(const cbm_rsq_code_t *c, float *out);
+
+#endif /* CBM_SEMANTIC_ROTSQ_H */

src/semantic/semantic.h

@@ -0,0 +1,214 @@
+/*
+ * semantic.h — Algorithmic code embeddings for SEMANTICALLY_RELATED edges.
+ *
+ * Combines 11 signals into a unified similarity score without external
+ * models or dependencies.  All signals derived from graph buffer metadata
+ * and the existing AST walk.
+ *
+ * Signals:
+ *   1. TF-IDF on metadata tokens (vocabulary overlap)
+ *   2. Random Indexing with co-occurrence (within-codebase synonym bridging)
+ *   3. MinHash structural (existing, decoded from "fp" property)
+ *   4. API Signature vectors (same callees → related)
+ *   5. Type Signature vectors (same param/return types → related)
+ *   6. Module Proximity (same directory → boost)
+ *   7. Decorator Pattern vectors (same annotations → related)
+ *   8. AST Structural Profile (control flow shape, expression types)
+ *   9. Approximate Data Flow (params→return, params→condition)
+ *  10. Graph Diffusion (transitive closure via neighbor blending)
+ *  11. Halstead-Lite (operator/operand complexity profile)
+ *
+ * Note: signals 8, 9, 11 are computed at extraction time (ast_profile.h)
+ * and stored in properties_json.  The rest are computed in the post-pass.
+ */
+#ifndef CBM_SEMANTIC_H
+#define CBM_SEMANTIC_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "semantic/rotsq.h"
+
+/* ── Configuration ───────────────────────────────────────────────── */
+
+/* Random Indexing dimension. 256 is sufficient for <500K functions. */
+/* 768 = nomic-embed-code embedding dimension.  Matches PRETRAINED_DIM. */
+enum { CBM_SEM_DIM = 768 };
+
+/* Random Indexing: non-zero entries per sparse random vector. */
+enum { CBM_SEM_SPARSE_NNZE = 8 };
+
+/* Co-occurrence window half-width. */
+enum { CBM_SEM_WINDOW = 5 };
+
+/* Frequent-token subsampling cap for co-occurrence enrichment. Code token
+ * frequencies are Zipfian — a handful of tokens (int, static, return, …) occur
+ * in 10^4–10^5 functions and dominate the O(occurrences × window × dim) corpus
+ * finalize cost. When a token has more than this many occurrences, we stride-
+ * sample down to ~this count: the enriched vector is L2-normalized afterward, so
+ * an evenly-spaced subsample preserves its *direction* while bounding the work.
+ * Rare/high-IDF (discriminative) tokens fall under the cap and are untouched.
+ * Mirrors word2vec/GloVe frequent-word subsampling. */
+enum { CBM_SEM_MAX_OCCUR = 512 };
+
+/* Default score threshold for SEMANTICALLY_RELATED edge emission.
+ * 0.75 balances recall with precision: validated ~95% precision on
+ * Linux kernel (0.80 = 100% but only 90 edges, 0.70 = 2047 edges
+ * but ~80% precision). */
+#define CBM_SEM_EDGE_THRESHOLD 0.75
+
+/* Maximum SEMANTICALLY_RELATED edges per node. */
+enum { CBM_SEM_MAX_EDGES = 10 };
+
+/* AST structural profile: 25 float features per function (control flow,
+ * nesting, expression types, literals, data flow, Halstead). */
+enum { CBM_SEM_AST_PROFILE_DIMS = 25 };
+
+/* MinHash fingerprint length (must match simhash/minhash.h CBM_MINHASH_K). */
+enum { CBM_SEM_MINHASH_K = 64 };
+
+/* Signal weights (sum to ~1.0, proximity is a multiplier). */
+typedef struct {
+    float w_tfidf;
+    float w_ri;
+    float w_minhash;
+    float w_api;
+    float w_type;
+    float w_decorator;
+    float w_struct_profile;
+    float w_dataflow;
+    float threshold;
+    int max_edges;
+} cbm_sem_config_t;
+
+/* Get default config (can be overridden via env vars). */
+cbm_sem_config_t cbm_sem_get_config(void);
+
+/* Check if semantic embeddings are enabled (CBM_SEMANTIC_ENABLED=1). */
+bool cbm_sem_is_enabled(void);
+
+/* ── Token extraction ────────────────────────────────────────────── */
+
+/* Maximum tokens per function from metadata (name + qn + path + sig + docstring + params). */
+enum { CBM_SEM_MAX_TOKENS = 512 };
+
+/* Split a name into tokens: camelCase, snake_case, dot.separated.
+ * Writes up to max_out tokens into out. Returns token count.
+ * Tokens are lowercased. Caller must free each token. */
+int cbm_sem_tokenize(const char *name, char **out, int max_out);
+
+/* ── Dense vectors ───────────────────────────────────────────────── */
+
+/* A fixed-size dense vector for cosine similarity. */
+typedef struct {
+    float v[CBM_SEM_DIM];
+} cbm_sem_vec_t;
+
+/* Compute cosine similarity between two dense vectors. */
+float cbm_sem_cosine(const cbm_sem_vec_t *a, const cbm_sem_vec_t *b);
+
+/* Generate a deterministic sparse random vector for a token.
+ * Uses xxHash(token) as seed. Output has SEM_SPARSE_NNZE non-zeros. */
+void cbm_sem_random_index(const char *token, cbm_sem_vec_t *out);
+
+/* Eagerly initialize the pretrained token lookup map.
+ * Call this BEFORE dispatching parallel work that invokes cbm_sem_random_index,
+ * so the lazy init races are avoided entirely on the hot path. */
+void cbm_sem_ensure_ready(void);
+
+/* Normalize a vector to unit length in-place. */
+void cbm_sem_normalize(cbm_sem_vec_t *v);
+
+/* Add src to dst: dst[i] += scale * src[i]. */
+void cbm_sem_vec_add_scaled(cbm_sem_vec_t *dst, const cbm_sem_vec_t *src, float scale);
+
+/* ── Per-function semantic data ──────────────────────────────────── */
+
+/* All computed signals for one function. */
+typedef struct {
+    int64_t node_id;
+    const char *file_path;
+    const char *file_ext;
+
+    /* Sparse TF-IDF: stored as parallel arrays of (token_index, weight). */
+    int *tfidf_indices;
+    float *tfidf_weights;
+    int tfidf_len;
+
+    /* Dense vectors for RI, API, Type, Decorator. */
+    /* Quantized semantic vectors (rotated 4-bit scalar quantization — see
+     * rotsq.h). The dense 768-float versions exist only transiently in the
+     * build workers: 4 x 3 KB resident floats per function were ~9.4 GB on
+     * the linux kernel; the codes are ~0.5 KB each. Scoring uses the exact
+     * code-expansion inner-product estimator (deterministic). */
+    cbm_rsq_code_t ri_code;
+    cbm_rsq_code_t api_code;
+    cbm_rsq_code_t type_code;
+    cbm_rsq_code_t deco_code;
+
+    /* AST profile as float vector (decoded from "sp" property). */
+    float struct_profile[CBM_SEM_AST_PROFILE_DIMS];
+
+    /* MinHash fingerprint (decoded from "fp" property). */
+    bool has_minhash;
+    uint32_t minhash[CBM_SEM_MINHASH_K];
+} cbm_sem_func_t;
+
+/* ── Corpus-level data ───────────────────────────────────────────── */
+
+/* Opaque corpus handle for IDF and Random Indexing state. */
+typedef struct cbm_sem_corpus cbm_sem_corpus_t;
+
+/* Create a new corpus from function data. */
+cbm_sem_corpus_t *cbm_sem_corpus_new(void);
+
+/* Register a function's tokens in the corpus (for IDF counting). */
+void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int count);
+
+/* Batch-build the corpus from pre-tokenized documents (PARALLEL variant).
+ * `all_tokens` layout: all_tokens[f * max_tokens_per_doc + t] = token pointer.
+ * `token_counts[f]` = number of tokens in document f.
+ * This replaces a loop of cbm_sem_corpus_add_doc() calls. */
+void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens,
+                                   const int *token_counts, int doc_count, int max_tokens_per_doc);
+
+/* Finalize: compute IDF, build enriched token vectors via co-occurrence. */
+void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus);
+
+/* Get IDF weight for a token. Returns 0.0 for unknown tokens. */
+float cbm_sem_corpus_idf(const cbm_sem_corpus_t *corpus, const char *token);
+
+/* Get the enriched Random Indexing vector for a token (after co-occurrence). */
+const cbm_sem_vec_t *cbm_sem_corpus_ri_vec(const cbm_sem_corpus_t *corpus, const char *token);
+
+/* Get the total document count. */
+int cbm_sem_corpus_doc_count(const cbm_sem_corpus_t *corpus);
+
+/* Get the total token count (vocabulary size). */
+int cbm_sem_corpus_token_count(const cbm_sem_corpus_t *corpus);
+
+/* Get token name and enriched vector by index (for serialization).
+ * Returns NULL if index is out of range. */
+const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index,
+                                    const cbm_sem_vec_t **out_vec, float *out_idf);
+
+/* Free corpus. */
+void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus);
+
+/* ── Combined scoring ────────────────────────────────────────────── */
+
+/* Compute combined similarity score between two functions. */
+float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
+                             const cbm_sem_config_t *cfg);
+
+/* Module proximity multiplier based on file paths. */
+float cbm_sem_proximity(const char *path_a, const char *path_b);
+
+/* ── Graph diffusion ─────────────────────────────────────────────── */
+
+/* Apply one iteration of graph diffusion to a combined embedding.
+ * Blends with mean of top-k neighbor embeddings (α=0.3). */
+void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, int neighbor_count,
+                     float alpha);
+
+#endif /* CBM_SEMANTIC_H */