wehub-resource-sync
1348 lines
54 KiB
C
1348 lines
54 KiB
C
#include "cbm.h"
|
|
#include "arena.h" // CBMArena, cbm_arena_init/alloc/strdup/destroy
|
|
#include "helpers.h"
|
|
#include "lang_specs.h"
|
|
#include "extract_unified.h"
|
|
#include "lsp/go_lsp.h"
|
|
#include "lsp/c_lsp.h"
|
|
#include "lsp/php_lsp.h"
|
|
#include "lsp/perl_lsp.h"
|
|
#include "lsp/py_lsp.h"
|
|
#include "lsp/ts_lsp.h"
|
|
#include "lsp/cs_lsp.h"
|
|
#include "lsp/java_lsp.h"
|
|
#include "lsp/kotlin_lsp.h"
|
|
#include "lsp/rust_lsp.h"
|
|
#include "preprocessor.h"
|
|
#include "foundation/compat.h"
|
|
#include "foundation/compat_fs.h" // cbm_fopen — crash-supervisor per-file marker write
|
|
#include "foundation/hash_table.h" // CBMHashTable — crash-supervisor quarantine set
|
|
#include "tree_sitter/api.h" // TSParser, TSNode, TSTree, TSInput, TSLanguage, TSPoint, TSParseOptions, TSParseState
|
|
#include "foundation/constants.h"
|
|
#include "mimalloc.h" // mi_malloc/mi_calloc/mi_realloc/mi_free/mi_usable_size — bind 3rd-party allocators (#424)
|
|
#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR
|
|
#include "sqlite3.h" // sqlite3_mem_methods, sqlite3_config, SQLITE_CONFIG_MALLOC — bind sqlite to mimalloc
|
|
#endif
|
|
#include <stdint.h> // uint32_t, uint64_t, int64_t
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <ctype.h>
|
|
#include <time.h> // struct timespec, CLOCK_MONOTONIC
|
|
|
|
// Atomic counters for profiling parse vs extraction time (nanoseconds).
|
|
// Accessed from multiple threads; using _Atomic for safe accumulation.
|
|
#include <stdatomic.h>
|
|
static _Atomic uint64_t total_parse_ns = 0;
|
|
static _Atomic uint64_t total_extract_ns = 0;
|
|
static _Atomic uint64_t total_lsp_ns = 0;
|
|
static _Atomic uint64_t total_preprocess_ns = 0;
|
|
static _Atomic uint64_t total_files_preprocessed = 0;
|
|
static _Atomic uint64_t total_files = 0;
|
|
|
|
// C/C++ preprocessor #define macros are extracted as Macro nodes (#375). On a
|
|
// macro-dense codebase (e.g. the Linux kernel: ~2.4M macros, 49% of all nodes)
|
|
// this is the dominant extraction cost, so it is gated to the full/advanced
|
|
// index modes. Default ON to preserve behavior for direct callers/tests; the
|
|
// pipeline sets it from the index mode before extraction. Set once pre-extract,
|
|
// read-only during, so a relaxed atomic is sufficient.
|
|
static _Atomic int g_extract_macros = 1;
|
|
void cbm_set_macro_extraction(int enabled) {
|
|
atomic_store_explicit(&g_extract_macros, enabled ? 1 : 0, memory_order_relaxed);
|
|
}
|
|
int cbm_macro_extraction_enabled(void) {
|
|
return atomic_load_explicit(&g_extract_macros, memory_order_relaxed);
|
|
}
|
|
|
|
#define NSEC_PER_SEC 1000000000ULL
|
|
#define USEC_TO_NSEC 1000ULL
|
|
/* Use compat.h's cbm_clock_gettime which accepts CLOCK_MONOTONIC (value
|
|
* varies by platform: 1 on Linux/Windows, 6 on macOS). We pass the
|
|
* platform value via the compat.h fallback. */
|
|
#if defined(CLOCK_MONOTONIC)
|
|
#define CBM_CLOCK_MONO CLOCK_MONOTONIC
|
|
#elif defined(__APPLE__)
|
|
#define CBM_CLOCK_MONO 6
|
|
#else
|
|
#define CBM_CLOCK_MONO 1
|
|
#endif
|
|
|
|
static uint64_t now_ns(void) {
|
|
struct timespec ts;
|
|
cbm_clock_gettime(CBM_CLOCK_MONO, &ts);
|
|
return ((uint64_t)ts.tv_sec * NSEC_PER_SEC) + (uint64_t)ts.tv_nsec;
|
|
}
|
|
|
|
// cbm_get_profile returns accumulated parse/extract times and file count.
|
|
void cbm_get_profile(cbm_profile_out_t out) {
|
|
*out.parse_ns = atomic_load(&total_parse_ns);
|
|
*out.extract_ns = atomic_load(&total_extract_ns);
|
|
*out.files = atomic_load(&total_files);
|
|
}
|
|
|
|
uint64_t cbm_get_lsp_ns(void) {
|
|
return atomic_load(&total_lsp_ns);
|
|
}
|
|
|
|
uint64_t cbm_get_preprocess_ns(void) {
|
|
return atomic_load(&total_preprocess_ns);
|
|
}
|
|
|
|
uint64_t cbm_get_files_preprocessed(void) {
|
|
return atomic_load(&total_files_preprocessed);
|
|
}
|
|
|
|
// cbm_reset_profile zeros the profiling counters.
|
|
void cbm_reset_profile(void) {
|
|
atomic_store(&total_parse_ns, 0);
|
|
atomic_store(&total_extract_ns, 0);
|
|
atomic_store(&total_lsp_ns, 0);
|
|
atomic_store(&total_preprocess_ns, 0);
|
|
atomic_store(&total_files_preprocessed, 0);
|
|
atomic_store(&total_files, 0);
|
|
}
|
|
|
|
// --- Growable array push functions ---
|
|
|
|
#define GROW_ARRAY(arr, arena) \
|
|
do { \
|
|
if ((arr)->count >= (arr)->cap) { \
|
|
int new_cap = (arr)->cap == 0 ? CBM_SZ_32 : (arr)->cap * PAIR_LEN; \
|
|
void *new_items = cbm_arena_alloc((arena), (size_t)new_cap * sizeof(*(arr)->items)); \
|
|
if (!new_items) \
|
|
return; \
|
|
if ((arr)->items && (arr)->count > 0) { \
|
|
memcpy(new_items, (arr)->items, (size_t)(arr)->count * sizeof(*(arr)->items)); \
|
|
} \
|
|
(arr)->items = new_items; \
|
|
(arr)->cap = new_cap; \
|
|
} \
|
|
} while (0)
|
|
|
|
void cbm_defs_push(CBMDefArray *arr, CBMArena *a, CBMDefinition def) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = def;
|
|
}
|
|
|
|
void cbm_calls_push(CBMCallArray *arr, CBMArena *a, CBMCall call) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = call;
|
|
}
|
|
|
|
void cbm_imports_push(CBMImportArray *arr, CBMArena *a, CBMImport imp) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = imp;
|
|
}
|
|
|
|
void cbm_usages_push(CBMUsageArray *arr, CBMArena *a, CBMUsage usage) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = usage;
|
|
}
|
|
|
|
void cbm_throws_push(CBMThrowArray *arr, CBMArena *a, CBMThrow thr) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = thr;
|
|
}
|
|
|
|
void cbm_rw_push(CBMRWArray *arr, CBMArena *a, CBMReadWrite rw) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = rw;
|
|
}
|
|
|
|
void cbm_typerefs_push(CBMTypeRefArray *arr, CBMArena *a, CBMTypeRef tr) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = tr;
|
|
}
|
|
|
|
void cbm_envaccess_push(CBMEnvAccessArray *arr, CBMArena *a, CBMEnvAccess ea) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = ea;
|
|
}
|
|
|
|
void cbm_typeassign_push(CBMTypeAssignArray *arr, CBMArena *a, CBMTypeAssign ta) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = ta;
|
|
}
|
|
|
|
void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = sr;
|
|
}
|
|
|
|
void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBinding ib) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = ib;
|
|
}
|
|
|
|
void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = it;
|
|
}
|
|
|
|
void cbm_resolvedcall_push(CBMResolvedCallArray *arr, CBMArena *a, CBMResolvedCall rc) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = rc;
|
|
}
|
|
|
|
void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch) {
|
|
GROW_ARRAY(arr, a);
|
|
arr->items[arr->count++] = ch;
|
|
}
|
|
|
|
// --- String input reader (for parse_with_options) ---
|
|
|
|
typedef struct {
|
|
const char *string;
|
|
uint32_t length;
|
|
} CBMStringInput;
|
|
|
|
static const char *cbm_string_read(void *payload, uint32_t byte, TSPoint point,
|
|
uint32_t *bytes_read) {
|
|
(void)point;
|
|
CBMStringInput *self = (CBMStringInput *)payload;
|
|
if (byte >= self->length) {
|
|
*bytes_read = 0;
|
|
return "";
|
|
}
|
|
*bytes_read = self->length - byte;
|
|
return self->string + byte;
|
|
}
|
|
|
|
// --- Parse timeout callback ---
|
|
|
|
static bool cbm_timeout_cb(TSParseState *state) {
|
|
uint64_t deadline = *(uint64_t *)state->payload;
|
|
return now_ns() > deadline;
|
|
}
|
|
|
|
// --- Thread-local parser pool ---
|
|
// TSParser is not thread-safe, but can be reused across files on the same thread.
|
|
// We keep one parser per thread, and just switch language as needed.
|
|
// This avoids ~70K ts_parser_new()/ts_parser_delete() cycles on large repos.
|
|
|
|
static CBM_TLS TSParser *tl_parser = NULL;
|
|
static CBM_TLS CBMLanguage tl_parser_lang = CBM_LANG_COUNT; // invalid sentinel
|
|
|
|
// Get or create a thread-local parser configured for the given language.
|
|
static TSParser *get_thread_parser(const TSLanguage *ts_lang, CBMLanguage lang) {
|
|
if (!tl_parser) {
|
|
tl_parser = ts_parser_new();
|
|
if (!tl_parser) {
|
|
return NULL;
|
|
}
|
|
tl_parser_lang = CBM_LANG_COUNT;
|
|
}
|
|
if (tl_parser_lang != lang) {
|
|
ts_parser_set_language(tl_parser, ts_lang);
|
|
tl_parser_lang = lang;
|
|
}
|
|
return tl_parser;
|
|
}
|
|
|
|
// --- Allocator binding (defense-in-depth, #424) ---
|
|
|
|
/* Bind tree-sitter and sqlite3 to mimalloc explicitly so a correct
|
|
* binary does NOT depend on the fragile MI_OVERRIDE symbol override. Under
|
|
* MI_OVERRIDE=1 — particularly the Windows static-MinGW link with
|
|
* --allow-multiple-definition — `malloc`/`free` can resolve to DIFFERENT
|
|
* allocators (mimalloc vs the CRT) inside third-party libs, so a block
|
|
* allocated by mimalloc gets freed by the CRT (or vice-versa), corrupting the
|
|
* heap freelist (#424). Binding each library through one explicit allocator
|
|
* eliminates that mismatch class generically, on every platform.
|
|
*
|
|
* Guarded to the production build (CBM_BIND_TS_ALLOCATOR=1, which CFLAGS_PROD
|
|
* defines alongside MI_OVERRIDE=1). The test build is CRT + ASan, where binding
|
|
* to mimalloc would mismatch ASan/CRT frees — there these binds compile to
|
|
* no-ops and the build stays unchanged. */
|
|
|
|
#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR
|
|
#include <assert.h>
|
|
|
|
/* sqlite3 mem methods backed by mimalloc. sqlite's xMalloc/xRealloc/xSize use
|
|
* `int` sizes; wrap with size_t casts. xRoundup rounds to an 8-byte boundary
|
|
* (sqlite requires 8-byte-aligned roundup, and mimalloc honors that alignment).
|
|
* Field order matches struct sqlite3_mem_methods exactly:
|
|
* xMalloc, xFree, xRealloc, xSize, xRoundup, xInit, xShutdown, pAppData. */
|
|
static void *cbm_sqlite_malloc(int n) {
|
|
return mi_malloc((size_t)n);
|
|
}
|
|
static void cbm_sqlite_free(void *p) {
|
|
mi_free(p);
|
|
}
|
|
static void *cbm_sqlite_realloc(void *p, int n) {
|
|
return mi_realloc(p, (size_t)n);
|
|
}
|
|
static int cbm_sqlite_size(void *p) {
|
|
return (int)mi_usable_size(p);
|
|
}
|
|
static int cbm_sqlite_roundup(int n) {
|
|
return (n + 7) & ~7; /* round up to 8-byte boundary */
|
|
}
|
|
static int cbm_sqlite_meminit(void *appdata) {
|
|
(void)appdata;
|
|
return SQLITE_OK;
|
|
}
|
|
static void cbm_sqlite_memshutdown(void *appdata) {
|
|
(void)appdata;
|
|
}
|
|
#endif /* CBM_BIND_TS_ALLOCATOR */
|
|
|
|
void cbm_alloc_init(void) {
|
|
#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR
|
|
static int alloc_bound = 0; /* single-threaded startup; plain int is fine */
|
|
if (alloc_bound) {
|
|
return;
|
|
}
|
|
alloc_bound = 1;
|
|
|
|
/* tree-sitter runtime (was previously bound in cbm_init; consolidated here). */
|
|
ts_set_allocator(mi_malloc, mi_calloc, mi_realloc, mi_free);
|
|
|
|
/* sqlite3. SQLITE_CONFIG_MALLOC MUST run before sqlite3_initialize / the
|
|
* first sqlite3_open* — otherwise sqlite3_config returns SQLITE_MISUSE
|
|
* silently and the binding is ignored. cbm_alloc_init() runs as the very
|
|
* first statement of main(), before cbm_mcp_server_new → cbm_store_open*. */
|
|
static sqlite3_mem_methods cbm_sqlite_mem = {
|
|
cbm_sqlite_malloc, /* xMalloc */
|
|
cbm_sqlite_free, /* xFree */
|
|
cbm_sqlite_realloc, /* xRealloc */
|
|
cbm_sqlite_size, /* xSize */
|
|
cbm_sqlite_roundup, /* xRoundup */
|
|
cbm_sqlite_meminit, /* xInit */
|
|
cbm_sqlite_memshutdown, /* xShutdown */
|
|
NULL, /* pAppData */
|
|
};
|
|
int sqlite_rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &cbm_sqlite_mem);
|
|
assert(sqlite_rc == SQLITE_OK && "SQLITE_CONFIG_MALLOC must run before sqlite3_initialize");
|
|
(void)sqlite_rc;
|
|
#endif /* CBM_BIND_TS_ALLOCATOR */
|
|
}
|
|
|
|
// --- Init/Shutdown ---
|
|
|
|
static int cbm_initialized = 0;
|
|
|
|
int cbm_init(void) {
|
|
if (cbm_initialized) {
|
|
return 0;
|
|
}
|
|
enum { CBM_INIT_DONE = 1 };
|
|
cbm_initialized = CBM_INIT_DONE;
|
|
/* Defense-in-depth allocator binds (idempotent). main() calls cbm_alloc_init
|
|
* first; this covers non-main entry points (pipeline passes call cbm_init).
|
|
* For sqlite the SQLITE_CONFIG_MALLOC bind only takes effect if it runs
|
|
* before sqlite initializes — main() guarantees that ordering; here it is a
|
|
* best-effort idempotent re-assert for paths that never hit main(). */
|
|
cbm_alloc_init();
|
|
return 0;
|
|
}
|
|
|
|
void cbm_reset_thread_parser(void) {
|
|
// Release parser's internal slab-allocated subtrees (stack, cached token).
|
|
// Must be called BEFORE cbm_slab_reset_thread() to avoid corrupting
|
|
// live slab chunks that the parser still references.
|
|
if (tl_parser) {
|
|
ts_parser_reset(tl_parser);
|
|
}
|
|
}
|
|
|
|
void cbm_destroy_thread_parser(void) {
|
|
// Full cleanup: delete the parser. Call on worker thread exit.
|
|
if (tl_parser) {
|
|
ts_parser_delete(tl_parser);
|
|
tl_parser = NULL;
|
|
tl_parser_lang = CBM_LANG_COUNT;
|
|
}
|
|
}
|
|
|
|
void cbm_shutdown(void) {
|
|
// Clean up thread-local parser for the calling thread.
|
|
// Note: other threads' TLS parsers are freed when those threads exit.
|
|
cbm_destroy_thread_parser();
|
|
cbm_initialized = 0;
|
|
}
|
|
|
|
// --- Bottleneck call-name classification (language-agnostic heuristics) ---
|
|
|
|
// Case-insensitive equality for short callee names.
|
|
static bool name_ieq(const char *a, const char *b) {
|
|
for (; *a && *b; a++, b++) {
|
|
if (tolower((unsigned char)*a) != tolower((unsigned char)*b)) {
|
|
return false;
|
|
}
|
|
}
|
|
return *a == '\0' && *b == '\0';
|
|
}
|
|
|
|
static bool name_in_set(const char *name, const char *const *set) {
|
|
for (const char *const *s = set; *s; s++) {
|
|
if (name_ieq(name, *s)) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Linear-scan / membership calls: a hit inside a loop is the textbook hidden
|
|
// O(n^2) (cf. Olivo et al., PLDI'15) that syntactic loop-depth alone misses.
|
|
static bool is_linear_scan_name(const char *n) {
|
|
static const char *const set[] = {"find", "indexof", "contains", "includes", "search",
|
|
"lookup", "strstr", "strchr", "strrchr", "memchr",
|
|
"find_if", "findindex", "count", "index", NULL};
|
|
return name_in_set(n, set);
|
|
}
|
|
|
|
// Allocation / growable-append calls: repeated inside a loop is the classic
|
|
// accidental reallocation / string-concat O(n^2). Names are deliberately
|
|
// conservative; meaningless in some languages → simply never matches there.
|
|
static bool is_alloc_name(const char *n) {
|
|
static const char *const set[] = {"malloc", "calloc", "realloc", "strdup", "strndup",
|
|
"append", "push_back", "emplace_back", "concat", "strcat",
|
|
"strncat", "push", "pushback", NULL};
|
|
return name_in_set(n, set);
|
|
}
|
|
|
|
// Extract the receiver identifier from a def's receiver text — Go's
|
|
// "(s *Store)" / "(s Store)" → "s". Stores the identifier start in *out and
|
|
// returns its length; returns 0 for unnamed receivers ("(*Store)", "(Store)"),
|
|
// where no second token follows the identifier (a lone token is the TYPE, not
|
|
// a name — such methods have no receiver variable to call through anyway).
|
|
static size_t receiver_ident(const char *recv_text, const char **out) {
|
|
const char *p = recv_text;
|
|
if (*p == '(') {
|
|
p++;
|
|
}
|
|
while (*p == ' ' || *p == '\t') {
|
|
p++;
|
|
}
|
|
const char *start = p;
|
|
while ((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || (*p >= '0' && *p <= '9') ||
|
|
*p == '_') {
|
|
p++;
|
|
}
|
|
size_t len = (size_t)(p - start);
|
|
if (len == 0) {
|
|
return 0; // "(*Store)": leading '*', no identifier
|
|
}
|
|
while (*p == ' ' || *p == '\t') {
|
|
p++;
|
|
}
|
|
if (*p == ')' || *p == '\0') {
|
|
return 0; // "(Store)": single token is the type, receiver unnamed
|
|
}
|
|
*out = start;
|
|
return len;
|
|
}
|
|
|
|
// Whether a callee expression targets the same instance/class as the enclosing
|
|
// def, i.e. counts as genuine self-recursion rather than a same-named call on a
|
|
// different receiver. callee_name may be bare ("recur") or qualified
|
|
// ("self.recur", "this.recur", "super().save", "axios.get", "self.obj.recur").
|
|
//
|
|
// Bare names have no receiver → assume self-call (free function calling itself
|
|
// by bare name; preserves prior behavior). Qualified names: the receiver chain
|
|
// is everything before the LAST '.', and the WHOLE chain must name the same
|
|
// object — self/this/cls/@self, or the enclosing def's own receiver identifier
|
|
// (Go: `s` in `func (s *Store) save()`, from CBMDefinition.receiver). Matching
|
|
// the whole chain (not its first segment) keeps self.obj.recur() out: it
|
|
// targets self's FIELD obj, a different object. super() is the parent class and
|
|
// any other receiver (axios, console, ...) a different target. See #599.
|
|
static bool is_self_receiver(const char *callee_name, const char *def_receiver) {
|
|
if (!callee_name || !callee_name[0]) {
|
|
return false;
|
|
}
|
|
const char *dot = strrchr(callee_name, '.');
|
|
if (!dot) {
|
|
return true; // bare name → self-recursion candidate
|
|
}
|
|
size_t rlen = (size_t)(dot - callee_name);
|
|
static const char *const self_receivers[] = {"self", "this", "cls", "@self", NULL};
|
|
for (int i = 0; self_receivers[i]; i++) {
|
|
size_t sl = strlen(self_receivers[i]);
|
|
if (rlen == sl && strncmp(callee_name, self_receivers[i], sl) == 0) {
|
|
return true;
|
|
}
|
|
}
|
|
if (def_receiver) {
|
|
const char *rid = NULL;
|
|
size_t ril = receiver_ident(def_receiver, &rid);
|
|
if (ril > 0 && ril == rlen && strncmp(callee_name, rid, ril) == 0) {
|
|
return true; // call through the enclosing method's own receiver
|
|
}
|
|
}
|
|
return false; // super() / axios / console / self.obj / any other receiver
|
|
}
|
|
|
|
// Count parameters from a signature string like "(int a, Foo* b, cb (*)(int,int))".
|
|
// Fallback for languages where param_names isn't populated (e.g. C keeps only the
|
|
// signature text). Counts commas at the top paren level; treats "()"/"(void)" as 0.
|
|
// Approximate by design (a structural smell, not an exact arity).
|
|
static int count_params_from_signature(const char *sig) {
|
|
if (!sig) {
|
|
return 0;
|
|
}
|
|
const char *p = sig;
|
|
while (*p && *p != '(') {
|
|
p++;
|
|
}
|
|
if (*p != '(') {
|
|
return 0;
|
|
}
|
|
p++;
|
|
const char *list = p;
|
|
int depth = 0;
|
|
int commas = 0;
|
|
bool any = false;
|
|
for (; *p; p++) {
|
|
char ch = *p;
|
|
if (ch == '(' || ch == '[' || ch == '{' || ch == '<') {
|
|
depth++;
|
|
} else if (ch == ')') {
|
|
if (depth == 0) {
|
|
break;
|
|
}
|
|
depth--;
|
|
} else if (ch == ']' || ch == '}' || ch == '>') {
|
|
if (depth > 0) {
|
|
depth--;
|
|
}
|
|
} else if (ch == ',' && depth == 0) {
|
|
commas++;
|
|
} else if (!isspace((unsigned char)ch)) {
|
|
any = true;
|
|
}
|
|
}
|
|
if (!any) {
|
|
return 0; /* "()" */
|
|
}
|
|
if (commas == 0) {
|
|
while (*list == ' ' || *list == '\t') {
|
|
list++;
|
|
}
|
|
if (strncmp(list, "void", 4) == 0 &&
|
|
(list[4] == ')' || list[4] == ' ' || list[4] == '\0')) {
|
|
return 0; /* C "(void)" */
|
|
}
|
|
}
|
|
return commas + 1;
|
|
}
|
|
|
|
// --- Main extraction function ---
|
|
|
|
/* Test-only deterministic fault injection for the crash/hang supervisor tests.
|
|
* Gated entirely behind env vars that are never set in production; a matching
|
|
* rel_path either aborts (a fault signal the supervisor classifies as a crash)
|
|
* or spins forever (an external-scanner infinite loop the quiet-timeout kills).
|
|
* This gives an honest guard — green iff the supervisor actually contains a real
|
|
* fault — instead of a fixture that may stop faulting once a root cause is fixed. */
|
|
/* Crash-supervisor per-file marker JOURNAL (Stage 3c skip-and-continue,
|
|
* parallel-safe). Recovery re-runs are PARALLEL (there are no sequential
|
|
* production runs), so a single overwrite-style marker would race across
|
|
* workers and — worse — go stale during non-extract phases, blaming
|
|
* whatever file was extracted LAST (that mis-quarantined four innocent
|
|
* ms-typescript fixtures, one 15-minute retry at a time). Instead every
|
|
* worker APPENDS one short line per event: "S <rel_path>" when it STARTS
|
|
* work on a file, "D <rel_path>" when it finishes it. A single short
|
|
* append of one line is atomic in practice on every target platform, and
|
|
* the parent discards a torn final line by design. The parent's suspect
|
|
* set after a crash/hang = files with an S but no D — exactly the
|
|
* in-flight set; a file is only quarantined after appearing in the
|
|
* suspect set of TWO CONSECUTIVE failed runs, so a stale or merely
|
|
* unlucky in-flight file is never quarantined alone. The env var is set
|
|
* solely by the supervisor during recovery — a no-op on normal runs. */
|
|
static void cbm_index_mark(const char *rel_path, char event) {
|
|
const char *mf = getenv("CBM_INDEX_MARKER_FILE");
|
|
if (!mf || !mf[0] || !rel_path || !rel_path[0]) {
|
|
return;
|
|
}
|
|
FILE *f = cbm_fopen(mf, "ab");
|
|
if (f) {
|
|
(void)fprintf(f, "%c %s\n", event, rel_path);
|
|
(void)fclose(f);
|
|
}
|
|
}
|
|
|
|
void cbm_index_mark_start(const char *rel_path) {
|
|
cbm_index_mark(rel_path, 'S');
|
|
}
|
|
|
|
void cbm_index_mark_done(const char *rel_path) {
|
|
cbm_index_mark(rel_path, 'D');
|
|
}
|
|
|
|
/* ── Crash-quarantine set (Stage 3c skip-and-continue) ──────────────────────
|
|
* After a crash the supervisor re-runs the worker single-threaded, passing
|
|
* CBM_INDEX_QUARANTINE_FILE — a newline-delimited list of repo-relative paths
|
|
* that already crashed the indexer and MUST NOT be extracted again. Owned here,
|
|
* next to the other env-driven extract hooks (marker + fault injector), so the
|
|
* single hard guard lives at the one choke point every pass funnels through
|
|
* (cbm_extract_file): whether a pass re-extracts from disk on a cache miss
|
|
* (sequential pass_calls/usages/semantic) or extracts fresh, a quarantined file
|
|
* short-circuits to an empty result and never reaches the parser/crash. The
|
|
* pipeline extract loops separately REPORT the skip as phase="crash" via
|
|
* cbm_index_is_quarantined() so the crasher surfaces in the response skipped[].
|
|
* Loaded once, lazily; read-only after load (safe for the parallel workers,
|
|
* though recovery runs single-threaded). Unset env ⇒ empty set ⇒ cheap no-op. */
|
|
static CBMHashTable *g_quarantine_set = NULL;
|
|
enum { CBM_QSET_UNINIT = 0, CBM_QSET_INITING = 1, CBM_QSET_INITED = 2 };
|
|
static atomic_int g_quarantine_state = CBM_QSET_UNINIT;
|
|
|
|
static void cbm_quarantine_load(void) {
|
|
const char *qf = getenv("CBM_INDEX_QUARANTINE_FILE");
|
|
if (!qf || !qf[0]) {
|
|
return; /* normal path: empty set */
|
|
}
|
|
FILE *f = cbm_fopen(qf, "rb");
|
|
if (!f) {
|
|
return;
|
|
}
|
|
CBMHashTable *set = cbm_ht_create(16);
|
|
if (!set) {
|
|
(void)fclose(f);
|
|
return;
|
|
}
|
|
char line[2048];
|
|
while (fgets(line, sizeof(line), f)) {
|
|
size_t len = strlen(line);
|
|
while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) {
|
|
line[--len] = '\0';
|
|
}
|
|
if (len == 0) {
|
|
continue;
|
|
}
|
|
/* Line format: "path\tphase" where phase is "crash" or "hang". A bare
|
|
* "path" line (no tab) is tolerated and defaults to phase "crash" for
|
|
* backward compatibility with older quarantine files. */
|
|
char *tab = strchr(line, '\t');
|
|
const char *phase = "crash";
|
|
if (tab) {
|
|
*tab = '\0';
|
|
if (tab[1]) {
|
|
phase = tab + 1;
|
|
}
|
|
}
|
|
if (line[0] == '\0') {
|
|
continue; /* empty path (line began with a tab) — skip */
|
|
}
|
|
/* The table borrows the key + value pointers, so dup both. Intentionally
|
|
* never freed: the set lives for the whole (short-lived worker) process.
|
|
* The value stores the phase so cbm_index_quarantine_phase() can report
|
|
* "crash" vs "hang"; membership (cbm_index_is_quarantined) is value != NULL. */
|
|
char *key = cbm_strdup(line);
|
|
char *pval = cbm_strdup(phase);
|
|
if (key && pval) {
|
|
cbm_ht_set(set, key, (void *)pval);
|
|
}
|
|
}
|
|
(void)fclose(f);
|
|
g_quarantine_set = set;
|
|
}
|
|
|
|
bool cbm_index_is_quarantined(const char *rel_path) {
|
|
if (!rel_path || !rel_path[0]) {
|
|
return false;
|
|
}
|
|
int state = atomic_load(&g_quarantine_state);
|
|
if (state != CBM_QSET_INITED) {
|
|
/* First caller wins the CAS and loads; racers spin until INITED.
|
|
* Same once-init pattern as cbm_ui_log_init (http_server.c). */
|
|
state = CBM_QSET_UNINIT;
|
|
if (atomic_compare_exchange_strong(&g_quarantine_state, &state, CBM_QSET_INITING)) {
|
|
cbm_quarantine_load();
|
|
atomic_store(&g_quarantine_state, CBM_QSET_INITED);
|
|
} else {
|
|
while (atomic_load(&g_quarantine_state) != CBM_QSET_INITED) {
|
|
cbm_usleep(1000); /* 1ms */
|
|
}
|
|
}
|
|
}
|
|
return g_quarantine_set && cbm_ht_has(g_quarantine_set, rel_path);
|
|
}
|
|
|
|
const char *cbm_index_quarantine_phase(const char *rel_path) {
|
|
/* cbm_index_is_quarantined drives the lazy once-load and returns true only
|
|
* when the set is loaded and holds rel_path — so on true, g_quarantine_set is
|
|
* non-NULL and the stored value is the phase string ("crash"/"hang"). */
|
|
if (!cbm_index_is_quarantined(rel_path)) {
|
|
return NULL;
|
|
}
|
|
return (const char *)cbm_ht_get(g_quarantine_set, rel_path);
|
|
}
|
|
|
|
static void cbm_test_fault_inject(const char *rel_path) {
|
|
if (!rel_path || !rel_path[0]) {
|
|
return;
|
|
}
|
|
const char *crash_on = getenv("CBM_TEST_CRASH_ON");
|
|
if (crash_on && crash_on[0] && strstr(rel_path, crash_on)) {
|
|
abort(); /* SIGABRT → WIFSIGNALED → classified as a crash */
|
|
}
|
|
const char *hang_on = getenv("CBM_TEST_HANG_ON");
|
|
if (hang_on && hang_on[0] && strstr(rel_path, hang_on)) {
|
|
for (;;) {
|
|
/* Busy-spin: the supervisor's quiet-timeout kills + reports us. */
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Pre-parse nesting guard for pathologically nested input. tree-sitter's GLR
|
|
* parser recurses once per nesting level inside stack_node_add_link
|
|
* (vendored ts_runtime/src/stack.c) while merging ambiguous parse-stack heads.
|
|
* The Perl grammar is genuinely ambiguous for `f(...)` (function call vs.
|
|
* bareword), so a deeply nested call chain `f(f(f(...)))` drives that recursion
|
|
* as deep as the nesting and overflows a small (1 MB Windows) stack *during the
|
|
* parse* — before any of the LSP walk-depth guards can fire. Unambiguous
|
|
* grammars (C/Java/Python) keep a single stack head and don't hit this, which is
|
|
* why only Perl crashed on the Windows/ARM CI runners.
|
|
*
|
|
* This is a workaround: the proper fix is bounding the GLR stack-merge recursion
|
|
* inside the vendored tree-sitter runtime, tracked upstream as #913. Remove this
|
|
* guard once that lands.
|
|
*
|
|
* cbm_source_nesting_exceeds scans the raw bytes for the maximum bracket-nesting
|
|
* depth and returns true as soon as it passes the cap (early-exit, O(n)). Real
|
|
* source never nests brackets this deep, so a file that does is skipped as a
|
|
* parse error (zero edges — graceful degradation, never a crash). Brackets in
|
|
* strings/comments are counted too: the only consequence of a false positive is
|
|
* skipping one absurd file, so string-awareness is not worth the cost. */
|
|
#define CBM_PERL_MAX_PARSE_NESTING 128
|
|
|
|
static bool cbm_source_nesting_exceeds(const char *source, int source_len, int cap) {
|
|
int depth = 0;
|
|
for (int i = 0; i < source_len; i++) {
|
|
char c = source[i];
|
|
if (c == '(' || c == '[' || c == '{') {
|
|
if (++depth > cap) {
|
|
return true;
|
|
}
|
|
} else if ((c == ')' || c == ']' || c == '}') && depth > 0) {
|
|
depth--;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len,
|
|
CBMLanguage language, const char *project,
|
|
const char *rel_path, int64_t timeout_micros,
|
|
const char **extra_defines, const char **include_paths);
|
|
|
|
/* Best-effort parse-coverage collection (#963). Walks only the has_error paths
|
|
* of the tree and records the 1-based line ranges of the TOP-MOST ERROR/MISSING
|
|
* nodes (does not descend into an error subtree — one range per failed region).
|
|
* Bounded by CBM_MAX_ERROR_REGIONS so pathological input can't blow up the
|
|
* output. The ranges mark where constructs were dropped; they are a detection
|
|
* aid, never a completeness proof. */
|
|
#define CBM_MAX_ERROR_REGIONS 64
|
|
typedef struct {
|
|
uint32_t starts[CBM_MAX_ERROR_REGIONS];
|
|
uint32_t ends[CBM_MAX_ERROR_REGIONS];
|
|
int count;
|
|
} cbm_error_regions_t;
|
|
|
|
static void cbm_error_regions_push(cbm_error_regions_t *acc, TSNode n) {
|
|
if (acc->count >= CBM_MAX_ERROR_REGIONS) {
|
|
return;
|
|
}
|
|
acc->starts[acc->count] = ts_node_start_point(n).row + 1;
|
|
acc->ends[acc->count] = ts_node_end_point(n).row + 1;
|
|
acc->count++;
|
|
}
|
|
|
|
static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc) {
|
|
if (acc->count >= CBM_MAX_ERROR_REGIONS) {
|
|
return;
|
|
}
|
|
uint32_t k = ts_node_child_count(n);
|
|
for (uint32_t i = 0; i < k && acc->count < CBM_MAX_ERROR_REGIONS; i++) {
|
|
TSNode c = ts_node_child(n, i);
|
|
if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) {
|
|
cbm_error_regions_push(acc, c); /* top-most region; do not descend */
|
|
} else if (ts_node_has_error(c)) {
|
|
cbm_collect_error_regions(c, acc);
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Recovery subtraction (#963): tree-sitter error recovery plus the
|
|
* ERROR-descending def walker often still extract constructs INSIDE a failed
|
|
* region (verified: a function in an #ifdef-split ERROR region and even a
|
|
* `def broken(:` both came back as defs). A region whose every line is
|
|
* covered by definitions that START inside it is definitely recovered — its
|
|
* constructs ARE in the graph — so flagging it would be a false miss.
|
|
* Container defs (Module/Package) are ignored: a file-spanning Module node is
|
|
* not evidence the region's constructs survived. Conservative: partially
|
|
* covered regions stay flagged. */
|
|
static bool cbm_region_is_recovered(uint32_t rs, uint32_t re, const CBMDefArray *defs) {
|
|
enum { MAX_COVER_DEFS = 256 };
|
|
uint32_t starts[MAX_COVER_DEFS];
|
|
uint32_t ends[MAX_COVER_DEFS];
|
|
int n = 0;
|
|
for (int i = 0; i < defs->count && n < MAX_COVER_DEFS; i++) {
|
|
const CBMDefinition *d = &defs->items[i];
|
|
if (!d->label || strcmp(d->label, "Module") == 0 || strcmp(d->label, "Package") == 0) {
|
|
continue;
|
|
}
|
|
if (d->start_line < rs || d->start_line > re) {
|
|
continue; /* recovery evidence must originate inside the region */
|
|
}
|
|
starts[n] = d->start_line;
|
|
ends[n] = d->end_line < d->start_line ? d->start_line : d->end_line;
|
|
n++;
|
|
}
|
|
if (n == 0) {
|
|
return false;
|
|
}
|
|
/* Insertion-sort by start, then sweep for gaps in [rs, re]. */
|
|
for (int i = 1; i < n; i++) {
|
|
uint32_t s = starts[i];
|
|
uint32_t e = ends[i];
|
|
int j = i - 1;
|
|
while (j >= 0 && starts[j] > s) {
|
|
starts[j + 1] = starts[j];
|
|
ends[j + 1] = ends[j];
|
|
j--;
|
|
}
|
|
starts[j + 1] = s;
|
|
ends[j + 1] = e;
|
|
}
|
|
uint32_t covered_to = rs - 1;
|
|
for (int i = 0; i < n; i++) {
|
|
if (starts[i] > covered_to + 1) {
|
|
return false; /* uncovered gap */
|
|
}
|
|
if (ends[i] > covered_to) {
|
|
covered_to = ends[i];
|
|
}
|
|
}
|
|
return covered_to >= re;
|
|
}
|
|
|
|
/* #961: true when 1-based `line` of `src` contains `name` (used to verify a
|
|
* def recovered from EXPANDED source really lives on that ORIGINAL line —
|
|
* rejects header-inlined defs whose physical expanded lines alias unrelated
|
|
* raw lines when compile_commands include paths are present). */
|
|
static bool cbm_line_contains(const char *src, int src_len, uint32_t line, const char *name) {
|
|
if (!src || !name || !name[0] || line == 0) {
|
|
return false;
|
|
}
|
|
uint32_t cur = 1;
|
|
int i = 0;
|
|
while (i < src_len && cur < line) {
|
|
if (src[i] == '\n') {
|
|
cur++;
|
|
}
|
|
i++;
|
|
}
|
|
if (cur != line) {
|
|
return false;
|
|
}
|
|
int end = i;
|
|
while (end < src_len && src[end] != '\n') {
|
|
end++;
|
|
}
|
|
size_t nlen = strlen(name);
|
|
for (int j = i; j + (int)nlen <= end; j++) {
|
|
if (strncmp(src + j, name, nlen) == 0) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static void cbm_subtract_recovered_regions(cbm_error_regions_t *regs, const CBMDefArray *defs) {
|
|
int kept = 0;
|
|
for (int i = 0; i < regs->count; i++) {
|
|
if (!cbm_region_is_recovered(regs->starts[i], regs->ends[i], defs)) {
|
|
regs->starts[kept] = regs->starts[i];
|
|
regs->ends[kept] = regs->ends[i];
|
|
kept++;
|
|
}
|
|
}
|
|
regs->count = kept;
|
|
}
|
|
|
|
/* Serialize collected regions as "start-end,start-end,..." into the arena. */
|
|
static const char *cbm_error_ranges_str(CBMArena *a, const cbm_error_regions_t *regs) {
|
|
if (regs->count <= 0) {
|
|
return NULL;
|
|
}
|
|
enum { RANGE_MAX = 24 }; /* "4294967295-4294967295," */
|
|
char *buf = (char *)cbm_arena_alloc(a, (size_t)regs->count * RANGE_MAX);
|
|
if (!buf) {
|
|
return NULL;
|
|
}
|
|
size_t off = 0;
|
|
for (int i = 0; i < regs->count; i++) {
|
|
off += (size_t)snprintf(buf + off, RANGE_MAX, "%s%u-%u", i ? "," : "", regs->starts[i],
|
|
regs->ends[i]);
|
|
}
|
|
return buf;
|
|
}
|
|
|
|
/* Public entry: run the extraction and journal completion. The DONE mark on
|
|
* every ordinary return (including error/timeout results) tells the crash
|
|
* supervisor this file did NOT kill the worker — only a file whose S has no
|
|
* D is a crash/hang suspect. */
|
|
CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage language,
|
|
const char *project, const char *rel_path, int64_t timeout_micros,
|
|
const char **extra_defines, const char **include_paths) {
|
|
CBMFileResult *r = cbm_extract_file_impl(source, source_len, language, project, rel_path,
|
|
timeout_micros, extra_defines, include_paths);
|
|
cbm_index_mark_done(rel_path);
|
|
return r;
|
|
}
|
|
|
|
static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len,
|
|
CBMLanguage language, const char *project,
|
|
const char *rel_path, int64_t timeout_micros,
|
|
const char **extra_defines,
|
|
const char **include_paths) {
|
|
// Allocate result on heap (arena inside for all string data)
|
|
enum { SINGLE = 1 };
|
|
CBMFileResult *result = (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult));
|
|
if (!result) {
|
|
return NULL;
|
|
}
|
|
|
|
cbm_arena_init(&result->arena);
|
|
CBMArena *a = &result->arena;
|
|
|
|
/* Crash-quarantine hard guard (Stage 3c): a file the supervisor pinned as a
|
|
* crasher must NEVER be parsed again. Return a clean empty result BEFORE the
|
|
* marker write and fault injector so no pass (including sequential re-extract
|
|
* passes that miss the result cache) can crash on it. The pipeline extract
|
|
* loops separately record it as a phase="crash" skip. Checked before the
|
|
* marker so quarantined files never overwrite it — the marker keeps pointing
|
|
* at the real (non-quarantined) file being processed when a crash hits. */
|
|
if (cbm_index_is_quarantined(rel_path)) {
|
|
return result;
|
|
}
|
|
|
|
cbm_index_mark_start(rel_path);
|
|
cbm_test_fault_inject(rel_path);
|
|
|
|
// Get language spec
|
|
const CBMLangSpec *spec = cbm_lang_spec(language);
|
|
if (!spec) {
|
|
result->has_error = true;
|
|
result->error_msg = cbm_arena_strdup(a, "unsupported language");
|
|
return result;
|
|
}
|
|
|
|
// Get tree-sitter language
|
|
const TSLanguage *ts_lang = cbm_ts_language(language);
|
|
if (!ts_lang) {
|
|
result->has_error = true;
|
|
result->error_msg = cbm_arena_strdup(a, "no tree-sitter grammar");
|
|
return result;
|
|
}
|
|
|
|
// Skip pathologically nested Perl before tree-sitter's recursive GLR stack
|
|
// merge overflows a small stack during the parse (see
|
|
// cbm_source_nesting_exceeds). Scoped to Perl: its ambiguous call grammar is
|
|
// the only one that drives that recursion to the nesting depth.
|
|
if (language == CBM_LANG_PERL &&
|
|
cbm_source_nesting_exceeds(source, source_len, CBM_PERL_MAX_PARSE_NESTING)) {
|
|
result->has_error = true;
|
|
result->error_msg = cbm_arena_strdup(a, "perl source nesting too deep; skipped");
|
|
return result;
|
|
}
|
|
|
|
// Get thread-local parser (reused across files on same thread)
|
|
TSParser *parser = get_thread_parser(ts_lang, language);
|
|
if (!parser) {
|
|
result->has_error = true;
|
|
result->error_msg = cbm_arena_strdup(a, "parser alloc failed");
|
|
return result;
|
|
}
|
|
|
|
// Reset parser state from any previous parse (cancellation flags etc.)
|
|
ts_parser_reset(parser);
|
|
|
|
uint64_t t0 = now_ns();
|
|
|
|
// Build string input + timeout options for parse_with_options
|
|
CBMStringInput str_input = {source, (uint32_t)source_len};
|
|
TSInput ts_input = {
|
|
&str_input,
|
|
cbm_string_read,
|
|
TSInputEncodingUTF8,
|
|
NULL,
|
|
};
|
|
|
|
TSParseOptions opts = {0};
|
|
uint64_t deadline_ns = 0; // cppcheck-suppress unreadVariable
|
|
if (timeout_micros > 0) {
|
|
deadline_ns = t0 + ((uint64_t)timeout_micros * USEC_TO_NSEC);
|
|
opts.payload = &deadline_ns;
|
|
opts.progress_callback = cbm_timeout_cb;
|
|
}
|
|
|
|
TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts);
|
|
uint64_t t1 = now_ns();
|
|
|
|
if (!tree) {
|
|
result->has_error = true;
|
|
result->error_msg =
|
|
cbm_arena_strdup(a, timeout_micros > 0 ? "parse timeout" : "parse failed");
|
|
return result;
|
|
}
|
|
|
|
TSNode root = ts_tree_root_node(tree);
|
|
|
|
// Compute module QN. Java/Go derive the module from the CONTAINING
|
|
// DIRECTORY (package semantics) rather than baking the filename stem in,
|
|
// so def QNs, the LSP caller_qn, and the textual calls-enclosing QN all
|
|
// agree (e.g. Outer.java -> module "proj", not "proj.Outer"). Other
|
|
// languages are unchanged.
|
|
result->module_qn = cbm_fqn_module_source_lang(a, project, rel_path, language);
|
|
result->is_test_file = cbm_is_test_file(rel_path, language);
|
|
|
|
// Build extraction context
|
|
CBMExtractCtx ctx = {
|
|
.arena = a,
|
|
.result = result,
|
|
.source = source,
|
|
.source_len = source_len,
|
|
.language = language,
|
|
.project = project,
|
|
.rel_path = rel_path,
|
|
.module_qn = result->module_qn,
|
|
.root = root,
|
|
};
|
|
|
|
// Run extractors: defs + imports use separate walks (unique recursion patterns),
|
|
// then a single unified cursor walk handles the remaining 7 extractors.
|
|
cbm_extract_definitions(&ctx);
|
|
cbm_extract_imports(&ctx);
|
|
cbm_extract_unified(&ctx);
|
|
|
|
// Channel detection (Socket.IO / EventEmitter) — JS/TS only.
|
|
cbm_extract_channels(&ctx);
|
|
|
|
// K8s / Kustomize semantic pass (additional structured extraction for YAML-based infra files).
|
|
if (ctx.language == CBM_LANG_KUSTOMIZE || ctx.language == CBM_LANG_K8S) {
|
|
cbm_extract_k8s(&ctx);
|
|
}
|
|
|
|
// LSP type-aware call/usage resolution (per-file). Runs in every mode;
|
|
// refines the tree-sitter + textual-resolution graph with type info.
|
|
uint64_t lsp_start = now_ns();
|
|
{
|
|
if (language == CBM_LANG_GO) {
|
|
cbm_run_go_lsp(a, result, source, source_len, root);
|
|
}
|
|
if (language == CBM_LANG_C || language == CBM_LANG_CPP || language == CBM_LANG_CUDA) {
|
|
cbm_run_c_lsp(a, result, source, source_len, root, language != CBM_LANG_C);
|
|
}
|
|
if (language == CBM_LANG_PHP) {
|
|
cbm_run_php_lsp(a, result, source, source_len, root);
|
|
}
|
|
if (language == CBM_LANG_PERL) {
|
|
cbm_run_perl_lsp(a, result, source, source_len, root);
|
|
}
|
|
if (language == CBM_LANG_PYTHON) {
|
|
cbm_run_py_lsp(a, result, source, source_len, root);
|
|
}
|
|
if (language == CBM_LANG_JAVASCRIPT || language == CBM_LANG_TYPESCRIPT ||
|
|
language == CBM_LANG_TSX) {
|
|
bool js_mode = (language == CBM_LANG_JAVASCRIPT);
|
|
// jsx_mode: TSX always; .jsx in the JS bucket also enables it.
|
|
bool jsx_mode = (language == CBM_LANG_TSX);
|
|
if (language == CBM_LANG_JAVASCRIPT && rel_path) {
|
|
size_t rl = strlen(rel_path);
|
|
if (rl >= 4 && strcmp(rel_path + rl - 4, ".jsx") == 0)
|
|
jsx_mode = true;
|
|
}
|
|
// dts_mode: ".d.ts" suffix (TypeScript only).
|
|
bool dts_mode = false;
|
|
if (language == CBM_LANG_TYPESCRIPT && rel_path) {
|
|
size_t rl = strlen(rel_path);
|
|
if (rl >= 5 && strcmp(rel_path + rl - 5, ".d.ts") == 0)
|
|
dts_mode = true;
|
|
}
|
|
cbm_run_ts_lsp(a, result, source, source_len, root, js_mode, jsx_mode, dts_mode);
|
|
}
|
|
if (language == CBM_LANG_CSHARP) {
|
|
cbm_run_cs_lsp(a, result, source, source_len, root);
|
|
}
|
|
}
|
|
if (language == CBM_LANG_JAVA) {
|
|
cbm_run_java_lsp(a, result, source, source_len, root);
|
|
}
|
|
if (language == CBM_LANG_KOTLIN) {
|
|
cbm_run_kotlin_lsp(a, result, source, source_len, root);
|
|
}
|
|
if (language == CBM_LANG_RUST) {
|
|
cbm_run_rust_lsp(a, result, source, source_len, root);
|
|
}
|
|
atomic_fetch_add(&total_lsp_ns, now_ns() - lsp_start);
|
|
|
|
// Calls extracted so far all carry ORIGINAL-source line numbers; the C/C++
|
|
// preprocessor second pass below appends calls with EXPANDED-source lines,
|
|
// which must not be used for the def line-range attribution of the bottleneck
|
|
// metrics. Remember the boundary.
|
|
int orig_calls_count = result->calls.count;
|
|
|
|
// Second pass: preprocess C/C++/CUDA and extract additional macro-hidden calls.
|
|
// Defs keep original-source line numbers; only CALLS are extracted from expanded source.
|
|
if (language == CBM_LANG_C || language == CBM_LANG_CPP || language == CBM_LANG_CUDA) {
|
|
uint64_t pp_start = now_ns();
|
|
char *expanded = cbm_preprocess(source, source_len, rel_path, extra_defines, include_paths,
|
|
language != CBM_LANG_C);
|
|
if (expanded) {
|
|
int expanded_len = (int)strlen(expanded);
|
|
// Record calls count before second pass
|
|
int calls_before = result->calls.count;
|
|
|
|
// Parse expanded source with fresh tree
|
|
TSParser *pp_parser = get_thread_parser(ts_lang, language);
|
|
if (pp_parser) {
|
|
ts_parser_reset(pp_parser);
|
|
CBMStringInput pp_input = {expanded, (uint32_t)expanded_len};
|
|
TSInput pp_ts_input = {
|
|
&pp_input,
|
|
cbm_string_read,
|
|
TSInputEncodingUTF8,
|
|
NULL,
|
|
};
|
|
TSParseOptions pp_opts = {0};
|
|
TSTree *pp_tree =
|
|
ts_parser_parse_with_options(pp_parser, NULL, pp_ts_input, pp_opts);
|
|
if (pp_tree) {
|
|
TSNode pp_root = ts_tree_root_node(pp_tree);
|
|
|
|
// Build context for expanded source — extract only calls via unified extractor
|
|
CBMExtractCtx pp_ctx = {
|
|
.arena = a,
|
|
.result = result,
|
|
.source = expanded,
|
|
.source_len = expanded_len,
|
|
.language = language,
|
|
.project = project,
|
|
.rel_path = rel_path,
|
|
.module_qn = result->module_qn,
|
|
.root = pp_root,
|
|
};
|
|
// Re-run unified extraction on expanded source.
|
|
// This adds macro-expanded calls; duplicates with original calls are
|
|
// harmless (pipeline deduplicates by caller+callee).
|
|
cbm_extract_unified(&pp_ctx);
|
|
|
|
// Also run LSP on expanded source for additional type-resolved
|
|
// calls (language is already C/C++/CUDA — checked in enclosing
|
|
// block). Runs in every mode.
|
|
cbm_run_c_lsp(a, result, expanded, expanded_len, pp_root,
|
|
language != CBM_LANG_C);
|
|
|
|
/* #961: a def whose body braces are split across
|
|
* #ifdef/#else branches parses as an ERROR region on the
|
|
* RAW source (both branches present at once -> unbalanced
|
|
* braces), so the raw defs walk silently dropped it. The
|
|
* expanded tree parses clean (simplecpp picked one
|
|
* branch) and same-file token lines stay aligned, so
|
|
* recover defs from it — adopting ONLY those that
|
|
* intersect a raw ERROR region, whose name is visible on
|
|
* the raw source line, and whose QN the raw pass did not
|
|
* already extract. */
|
|
if (ts_node_has_error(root)) {
|
|
cbm_error_regions_t raw_regs = {{0}, {0}, 0};
|
|
cbm_collect_error_regions(root, &raw_regs);
|
|
if (raw_regs.count > 0) {
|
|
int defs_before = result->defs.count;
|
|
cbm_extract_definitions(&pp_ctx);
|
|
int w = defs_before;
|
|
for (int i = defs_before; i < result->defs.count; i++) {
|
|
CBMDefinition *d = &result->defs.items[i];
|
|
bool adopt = false;
|
|
for (int rj = 0; rj < raw_regs.count && !adopt; rj++) {
|
|
if (d->start_line <= raw_regs.ends[rj] &&
|
|
d->end_line >= raw_regs.starts[rj]) {
|
|
adopt = true;
|
|
}
|
|
}
|
|
if (adopt &&
|
|
(!d->name || !cbm_line_contains(source, source_len,
|
|
d->start_line, d->name))) {
|
|
adopt = false;
|
|
}
|
|
for (int j = 0; j < defs_before && adopt; j++) {
|
|
const char *q = result->defs.items[j].qualified_name;
|
|
if (q && d->qualified_name &&
|
|
strcmp(q, d->qualified_name) == 0) {
|
|
adopt = false;
|
|
}
|
|
}
|
|
if (adopt) {
|
|
result->defs.items[w++] = *d;
|
|
}
|
|
}
|
|
result->defs.count = w;
|
|
}
|
|
}
|
|
|
|
ts_tree_delete(pp_tree);
|
|
}
|
|
}
|
|
cbm_preprocess_free(expanded);
|
|
atomic_fetch_add(&total_files_preprocessed, 1);
|
|
(void)calls_before; // used for future logging
|
|
}
|
|
atomic_fetch_add(&total_preprocess_ns, now_ns() - pp_start);
|
|
}
|
|
|
|
// Bottleneck call-context metrics. Each call is attributed to the INNERMOST
|
|
// enclosing Function/Method def by source-line range (defs and calls in one
|
|
// CBMFileResult share the same file). Range matching is used instead of
|
|
// enclosing_func_qn string matching because some grammars (notably C, whose
|
|
// function_definition has no "name" field) attribute the call's scope to the
|
|
// module rather than the function — line ranges are unambiguous and
|
|
// language-agnostic. Bounded per file (defs x calls), not a repo-scale scan.
|
|
int def_count = result->defs.count;
|
|
bool *has_self = def_count > 0 ? calloc((size_t)def_count, sizeof(bool)) : NULL;
|
|
bool *has_guarded = def_count > 0 ? calloc((size_t)def_count, sizeof(bool)) : NULL;
|
|
|
|
// param_count is a standalone structural smell (independent of calls). Prefer
|
|
// the parsed param_names array; fall back to counting from the signature text
|
|
// for languages (e.g. C) that populate only the signature.
|
|
for (int di = 0; di < def_count; di++) {
|
|
CBMDefinition *d = &result->defs.items[di];
|
|
int pc = 0;
|
|
if (d->param_names) {
|
|
while (d->param_names[pc]) {
|
|
pc++;
|
|
}
|
|
}
|
|
if (pc == 0 && d->signature) {
|
|
pc = count_params_from_signature(d->signature);
|
|
}
|
|
d->param_count = pc;
|
|
}
|
|
|
|
for (int ci = 0; ci < orig_calls_count; ci++) {
|
|
const CBMCall *c = &result->calls.items[ci];
|
|
if (!c->callee_name || c->start_line <= 0) {
|
|
continue;
|
|
}
|
|
// Innermost enclosing Function/Method def by line range (smallest span).
|
|
int best = -1;
|
|
int best_span = -1;
|
|
for (int di = 0; di < def_count; di++) {
|
|
const CBMDefinition *d = &result->defs.items[di];
|
|
if (!d->name || !d->label ||
|
|
(strcmp(d->label, "Function") != 0 && strcmp(d->label, "Method") != 0)) {
|
|
continue;
|
|
}
|
|
if ((int)d->start_line <= c->start_line && c->start_line <= (int)d->end_line) {
|
|
int span = (int)d->end_line - (int)d->start_line;
|
|
if (best < 0 || span < best_span) {
|
|
best_span = span;
|
|
best = di;
|
|
}
|
|
}
|
|
}
|
|
if (best < 0) {
|
|
continue;
|
|
}
|
|
CBMDefinition *d = &result->defs.items[best];
|
|
// callee_name may be bare ("recur") or qualified ("self.recur",
|
|
// "super().save", "axios.get"). A short-name match alone is not
|
|
// self-recursion: the callee must also target the same object
|
|
// (is_self_receiver), or super().save() inside save and axios.get
|
|
// inside get are false positives (#599).
|
|
const char *dot = strrchr(c->callee_name, '.');
|
|
const char *callee_short = dot ? dot + 1 : c->callee_name;
|
|
bool in_loop = c->loop_depth > 0;
|
|
|
|
if (strcmp(callee_short, d->name) == 0 && is_self_receiver(c->callee_name, d->receiver)) {
|
|
// Direct self-recursion. The call graph omits self-edges (pass_calls
|
|
// skips source==target), so detect it here; seeds "recursive".
|
|
d->is_recursive = true;
|
|
if (has_self) {
|
|
has_self[best] = true;
|
|
}
|
|
if (in_loop) {
|
|
d->recursion_in_loop = true; // recursion compounded by a loop
|
|
}
|
|
if (c->branch_depth > 0 && has_guarded) {
|
|
has_guarded[best] = true; // a self-call guarded by some conditional
|
|
}
|
|
}
|
|
if (in_loop && is_linear_scan_name(callee_short)) {
|
|
d->linear_scan_in_loop++; // hidden O(n^2): linear scan inside a loop
|
|
}
|
|
if (in_loop && is_alloc_name(callee_short)) {
|
|
d->alloc_in_loop++; // repeated allocation/append inside a loop
|
|
}
|
|
}
|
|
|
|
// Recursive with no self-call guarded by any conditional → no obvious base
|
|
// case on the recursive path: a stronger "potentially unbounded" signal.
|
|
for (int di = 0; di < def_count; di++) {
|
|
if (has_self && has_self[di] && !(has_guarded && has_guarded[di])) {
|
|
result->defs.items[di].unguarded_recursion = true;
|
|
}
|
|
}
|
|
free(has_self);
|
|
free(has_guarded);
|
|
|
|
uint64_t t2 = now_ns();
|
|
|
|
/* Best-effort parse-coverage signal (#963): flag files whose tree contains
|
|
* ERROR/MISSING regions. Computed AFTER extraction so definite recovery is
|
|
* subtracted first — a region fully re-extracted as definitions is not a
|
|
* miss, and a fully recovered file is not flagged at all. Detection aid
|
|
* only: the absence of this flag is NOT a completeness guarantee. */
|
|
if (ts_node_has_error(root)) {
|
|
cbm_error_regions_t regs = {{0}, {0}, 0};
|
|
if (strcmp(ts_node_type(root), "ERROR") == 0) {
|
|
cbm_error_regions_push(®s, root); /* whole file unparseable */
|
|
} else {
|
|
cbm_collect_error_regions(root, ®s);
|
|
}
|
|
cbm_subtract_recovered_regions(®s, &result->defs);
|
|
if (regs.count > 0) {
|
|
result->parse_incomplete = true;
|
|
result->error_region_count = regs.count;
|
|
result->error_ranges = cbm_error_ranges_str(a, ®s);
|
|
}
|
|
}
|
|
|
|
result->imports_count = result->imports.count;
|
|
|
|
// Accumulate profiling counters
|
|
atomic_fetch_add(&total_parse_ns, t1 - t0);
|
|
atomic_fetch_add(&total_extract_ns, t2 - t1);
|
|
atomic_fetch_add(&total_files, 1);
|
|
|
|
// Retain tree for cross-file LSP reuse (caller frees via cbm_free_tree)
|
|
result->cached_tree = tree;
|
|
result->cached_lang = language;
|
|
return result;
|
|
}
|
|
|
|
void cbm_free_result(CBMFileResult *result) {
|
|
if (!result) {
|
|
return;
|
|
}
|
|
if (result->cached_tree) {
|
|
ts_tree_delete(result->cached_tree);
|
|
result->cached_tree = NULL;
|
|
}
|
|
cbm_arena_destroy(&result->arena);
|
|
free(result);
|
|
}
|
|
|
|
void cbm_free_tree(CBMFileResult *result) {
|
|
if (result && result->cached_tree) {
|
|
ts_tree_delete(result->cached_tree);
|
|
result->cached_tree = NULL;
|
|
}
|
|
}
|
|
|
|
void cbm_free_tree_ptr(TSTree *tree) {
|
|
if (tree) {
|
|
ts_tree_delete(tree);
|
|
}
|
|
}
|