chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+116
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# audit-grammar-security.sh — Scan vendored tree-sitter grammar scanner.c
|
||||
# files for dangerous patterns that could indicate malicious code.
|
||||
#
|
||||
# Grammars are third-party C source compiled into our binary. parser.c files
|
||||
# are auto-generated (low risk), but scanner.c files are hand-written C with
|
||||
# access to memory allocation and input text.
|
||||
#
|
||||
# Usage: scripts/audit-grammar-security.sh [directory]
|
||||
# Default: internal/cbm/vendored/grammars/
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
GRAMMAR_DIR="${1:-$ROOT/internal/cbm/vendored/grammars}"
|
||||
|
||||
if [ ! -d "$GRAMMAR_DIR" ]; then
|
||||
echo "ERROR: directory not found: $GRAMMAR_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Grammar Security Audit ==="
|
||||
echo "Scanning: $GRAMMAR_DIR"
|
||||
echo ""
|
||||
|
||||
TOTAL=0
|
||||
WARNED=0
|
||||
BLOCKED=0
|
||||
|
||||
# Safe includes that scanners legitimately use
|
||||
SAFE_INCLUDES='string\.h|stdint\.h|stdbool\.h|stdlib\.h|wctype\.h|stdio\.h|stddef\.h|limits\.h|assert\.h|ctype\.h|wchar\.h|math\.h|stdalign\.h|stdarg\.h|float\.h|inttypes\.h'
|
||||
|
||||
# Dangerous includes
|
||||
DANGER_INCLUDES='unistd\.h|sys/|netdb\.h|dlfcn\.h|signal\.h|spawn\.h|pthread\.h|fcntl\.h|dirent\.h|termios\.h|arpa/|netinet/'
|
||||
|
||||
# Dangerous function calls (word-boundary \b to avoid matching lex_accept, etc.)
|
||||
# getenv("TREE_SITTER_DEBUG") is a standard tree-sitter pattern, excluded.
|
||||
DANGER_CALLS='\bsystem\s*\(|\bexec[lvpe]+\s*\(|\bpopen\s*\(|\bfopen\s*\(|\bfwrite\s*\(|\bsocket\s*\(|\bfork\s*\(|\bdlopen\s*\(|\bconnect\s*\(|\bbind\s*\(|\blisten\s*\(|\bsendto\s*\(|\brecvfrom\s*\(|\bmmap\s*\(|\bmprotect\s*\('
|
||||
# getenv is only dangerous for non-debug uses
|
||||
DANGER_GETENV='\bgetenv\s*\('
|
||||
SAFE_GETENV='TREE_SITTER_DEBUG'
|
||||
|
||||
# Suspicious patterns
|
||||
SUSPICIOUS='__attribute__\s*\(\s*\(\s*constructor|__attribute__\s*\(\s*\(\s*destructor|asm\s*\(|__asm__|__asm\b'
|
||||
|
||||
for grammar_dir in "$GRAMMAR_DIR"/*/; do
|
||||
name=$(basename "$grammar_dir")
|
||||
TOTAL=$((TOTAL + 1))
|
||||
issues=""
|
||||
|
||||
for src in "$grammar_dir"scanner.c "$grammar_dir"scanner.cc "$grammar_dir"*.h "$grammar_dir"*.inc; do
|
||||
[ -f "$src" ] || continue
|
||||
basename_src=$(basename "$src")
|
||||
|
||||
# Skip tree_sitter/ subdirectory headers (those are standard)
|
||||
case "$src" in
|
||||
*/tree_sitter/*) continue ;;
|
||||
esac
|
||||
|
||||
# Check dangerous includes
|
||||
while IFS= read -r line; do
|
||||
issues="${issues} WARN $basename_src: dangerous include: $line\n"
|
||||
done < <(grep -nE "#include\s*<($DANGER_INCLUDES)" "$src" 2>/dev/null || true)
|
||||
|
||||
# Check dangerous calls
|
||||
while IFS= read -r line; do
|
||||
issues="${issues} BLOCK $basename_src: dangerous call: $line\n"
|
||||
done < <(grep -nE "$DANGER_CALLS" "$src" 2>/dev/null || true)
|
||||
|
||||
# Check getenv separately (allow TREE_SITTER_DEBUG)
|
||||
while IFS= read -r line; do
|
||||
if ! echo "$line" | grep -qF "$SAFE_GETENV"; then
|
||||
issues="${issues} BLOCK $basename_src: dangerous call: $line\n"
|
||||
fi
|
||||
done < <(grep -nE "$DANGER_GETENV" "$src" 2>/dev/null || true)
|
||||
|
||||
# Check suspicious patterns
|
||||
while IFS= read -r line; do
|
||||
issues="${issues} BLOCK $basename_src: suspicious pattern: $line\n"
|
||||
done < <(grep -nE "$SUSPICIOUS" "$src" 2>/dev/null || true)
|
||||
|
||||
# Check for base64-like long encoded strings (40+ alphanumeric chars)
|
||||
while IFS= read -r line; do
|
||||
issues="${issues} WARN $basename_src: possible encoded data: $line\n"
|
||||
done < <(grep -nE '"[A-Za-z0-9+/]{60,}={0,2}"' "$src" 2>/dev/null || true)
|
||||
done
|
||||
|
||||
if [ -n "$issues" ]; then
|
||||
if echo -e "$issues" | grep -q "BLOCK"; then
|
||||
echo "BLOCK $name:"
|
||||
echo -e "$issues"
|
||||
BLOCKED=$((BLOCKED + 1))
|
||||
else
|
||||
echo "WARN $name:"
|
||||
echo -e "$issues"
|
||||
WARNED=$((WARNED + 1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "────────────────────────────────────────────"
|
||||
echo " Scanned: $TOTAL grammars"
|
||||
echo " Clean: $((TOTAL - WARNED - BLOCKED))"
|
||||
echo " Warned: $WARNED"
|
||||
echo " Blocked: $BLOCKED"
|
||||
echo "────────────────────────────────────────────"
|
||||
|
||||
if [ "$BLOCKED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "FAILED: $BLOCKED grammar(s) have dangerous patterns. Review before vendoring."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "PASSED: No dangerous patterns found."
|
||||
exit 0
|
||||
Executable
+226
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Byte-identity audit: compare every vendored license file against upstream.
|
||||
|
||||
Verdicts:
|
||||
IDENTICAL byte-equal to the upstream default-branch license
|
||||
IDENTICAL@PINNED byte-equal to the license at the manifest's pinned commit
|
||||
FIRST-PARTY-OK byte-equal to the project root LICENSE
|
||||
FIRST-PARTY-VAR first-party but text differs from root LICENSE (inspect)
|
||||
DIFFERS no byte-equal upstream candidate found (inspect)
|
||||
ERROR upstream fetch failed
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
GRAMMARS = os.path.join(ROOT, "internal/cbm/vendored/grammars")
|
||||
|
||||
FIRST_PARTY = {"cobol", "form", "janet", "magma", "protobuf", "wolfram"}
|
||||
FORKS = { # self-maintained forks: vendored LICENSE must match the original upstream
|
||||
"cfml": "cfmleditor/tree-sitter-cfml",
|
||||
"cfscript": "cfmleditor/tree-sitter-cfml",
|
||||
"dotenv": "pnx/tree-sitter-dotenv",
|
||||
"qml": "yuja/tree-sitter-qmljs",
|
||||
}
|
||||
SPECIAL_NOTICE = {
|
||||
"assembly": "RETAINED-MIT (upstream RubixDev/tree-sitter-assembly deleted from GitHub)",
|
||||
"pine": "PROVENANCE-NOTICE (upstream kvarenzn/tree-sitter-pine declares ISC, ships no license file)",
|
||||
}
|
||||
DISAGREEMENT = {
|
||||
"jinja2": "dbt-labs/tree-sitter-jinja2",
|
||||
"just": "casey/tree-sitter-just",
|
||||
"move": "tzakian/tree-sitter-move",
|
||||
"sshconfig": "ObserverOfTime/tree-sitter-ssh-config",
|
||||
"zsh": "georgeharker/tree-sitter-zsh",
|
||||
}
|
||||
LIBS = {
|
||||
"vendored/mimalloc": ("microsoft/mimalloc", None),
|
||||
"vendored/tre": ("laurikari/tre", None),
|
||||
"vendored/xxhash": ("Cyan4973/xxHash", None),
|
||||
"vendored/yyjson": ("ibireme/yyjson", None),
|
||||
"internal/cbm/vendored/lz4": ("lz4/lz4", "lib/LICENSE"),
|
||||
"internal/cbm/vendored/zstd": ("facebook/zstd", None),
|
||||
"internal/cbm/vendored/simplecpp": ("danmar/simplecpp", None),
|
||||
"internal/cbm/vendored/verstable": ("JacksonAllan/Verstable", None),
|
||||
"internal/cbm/vendored/wyhash": ("wangyi-fudan/wyhash", None),
|
||||
"internal/cbm/vendored/ts_runtime": ("tree-sitter/tree-sitter", None),
|
||||
"internal/cbm/vendored/common": ("tree-sitter/tree-sitter-html", None),
|
||||
"internal/cbm/vendored/common/tree_sitter": ("tree-sitter/tree-sitter", None),
|
||||
}
|
||||
CANDIDATE_NAMES = ["LICENSE", "LICENSE.md", "LICENSE.txt", "COPYING",
|
||||
"COPYING.txt", "LICENSE-MIT", "UNLICENSE", "LICENCE",
|
||||
"license", "License.txt", "NOTICE"]
|
||||
|
||||
|
||||
def gh_api(path):
|
||||
r = subprocess.run(["gh", "api", path], capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
return r.stdout
|
||||
|
||||
|
||||
def upstream_default_license(repo):
|
||||
out = gh_api(f"repos/{repo}/license")
|
||||
if not out:
|
||||
return None
|
||||
try:
|
||||
d = json.loads(out)
|
||||
return base64.b64decode(d.get("content", "")).decode("utf-8", "replace")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def upstream_file(repo, path, ref=None):
|
||||
url = f"repos/{repo}/contents/{path}"
|
||||
if ref:
|
||||
url += f"?ref={ref}"
|
||||
out = gh_api(url)
|
||||
if not out:
|
||||
return None
|
||||
try:
|
||||
d = json.loads(out)
|
||||
if isinstance(d, dict) and d.get("content"):
|
||||
return base64.b64decode(d["content"]).decode("utf-8", "replace")
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def local_license(dirpath):
|
||||
if not os.path.isdir(dirpath):
|
||||
return None, None
|
||||
for f in sorted(os.listdir(dirpath)):
|
||||
if re.match(r"^(LICENSE|LICENCE|COPYING|UNLICENSE)", f, re.I):
|
||||
p = os.path.join(dirpath, f)
|
||||
with open(p, encoding="utf-8", errors="replace") as fh:
|
||||
return f, fh.read()
|
||||
return None, None
|
||||
|
||||
|
||||
def parse_manifest():
|
||||
"""grammar -> (repo, pinned_commit) from the verified-upstream table."""
|
||||
out = {}
|
||||
with open(os.path.join(GRAMMARS, "MANIFEST.md"), encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
m = re.match(
|
||||
r"^\| (\w[\w.+-]*) \| \d+ \| ([\w.-]+/[\w.-]+) \| `([^`]+)` \|", line)
|
||||
if m:
|
||||
out[m.group(1)] = (m.group(2), m.group(3))
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
with open(os.path.join(ROOT, "LICENSE"), encoding="utf-8") as fh:
|
||||
root_license = fh.read()
|
||||
|
||||
manifest = parse_manifest()
|
||||
results = {}
|
||||
|
||||
def check_upstream(key, dirpath, repo, pinned=None, exact_path=None):
|
||||
fname, ours = local_license(dirpath)
|
||||
if ours is None:
|
||||
results[key] = ("NO-LOCAL-LICENSE", "")
|
||||
return
|
||||
# 1) exact upstream path (e.g. lz4 lib/LICENSE), at HEAD then pinned
|
||||
if exact_path:
|
||||
for ref in (None, pinned):
|
||||
up = upstream_file(repo, exact_path, ref)
|
||||
if up is not None and up == ours:
|
||||
results[key] = ("IDENTICAL" if ref is None else "IDENTICAL@PINNED",
|
||||
f"{repo}:{exact_path}")
|
||||
return
|
||||
# 2) default-branch detected license
|
||||
up = upstream_default_license(repo)
|
||||
if up is not None and up == ours:
|
||||
results[key] = ("IDENTICAL", f"{repo} (default branch)")
|
||||
return
|
||||
# 3) pinned-commit candidates by filename
|
||||
if pinned:
|
||||
tried = [fname] + [c for c in CANDIDATE_NAMES if c != fname]
|
||||
for cand in tried:
|
||||
up2 = upstream_file(repo, cand, pinned)
|
||||
if up2 is not None and up2 == ours:
|
||||
results[key] = ("IDENTICAL@PINNED", f"{repo}:{cand}@{pinned}")
|
||||
return
|
||||
# 4) HEAD candidates by filename
|
||||
for cand in CANDIDATE_NAMES:
|
||||
up3 = upstream_file(repo, cand)
|
||||
if up3 is not None and up3 == ours:
|
||||
results[key] = ("IDENTICAL", f"{repo}:{cand} (default branch)")
|
||||
return
|
||||
results[key] = ("DIFFERS" if up is not None else "ERROR", f"{repo}")
|
||||
|
||||
# Libraries
|
||||
for rel, (repo, exact) in LIBS.items():
|
||||
check_upstream(rel, os.path.join(ROOT, rel), repo, exact_path=exact)
|
||||
|
||||
# Special: nomic = canonical Apache-2.0 text; sqlite3 = first-party notice
|
||||
fname, ours = local_license(os.path.join(ROOT, "vendored/nomic"))
|
||||
apache = subprocess.run(
|
||||
["curl", "-fsSL", "https://www.apache.org/licenses/LICENSE-2.0.txt"],
|
||||
capture_output=True, text=True).stdout
|
||||
results["vendored/nomic"] = (
|
||||
"IDENTICAL" if ours == apache else "DIFFERS",
|
||||
"apache.org canonical LICENSE-2.0.txt")
|
||||
results["vendored/sqlite3"] = ("FIRST-PARTY-NOTICE",
|
||||
"own public-domain notice (sqlite has no upstream LICENSE)")
|
||||
|
||||
# Grammars
|
||||
for g in sorted(os.listdir(GRAMMARS)):
|
||||
d = os.path.join(GRAMMARS, g)
|
||||
if not os.path.isdir(d):
|
||||
continue
|
||||
key = f"grammars/{g}"
|
||||
if g in FIRST_PARTY:
|
||||
fname, ours = local_license(d)
|
||||
if ours == root_license:
|
||||
results[key] = ("FIRST-PARTY-OK", "== project root LICENSE")
|
||||
else:
|
||||
results[key] = ("FIRST-PARTY-VAR", f"{fname}: differs from root LICENSE")
|
||||
continue
|
||||
if g in FORKS:
|
||||
check_upstream(key, d, FORKS[g])
|
||||
continue
|
||||
if g in SPECIAL_NOTICE:
|
||||
results[key] = ("MANUAL-VERIFIED", SPECIAL_NOTICE[g])
|
||||
continue
|
||||
if g in DISAGREEMENT:
|
||||
check_upstream(key, d, DISAGREEMENT[g])
|
||||
continue
|
||||
if g in manifest:
|
||||
repo, pinned = manifest[g]
|
||||
check_upstream(key, d, repo, pinned=pinned)
|
||||
continue
|
||||
results[key] = ("NO-MANIFEST-ENTRY", "")
|
||||
|
||||
# Report
|
||||
from collections import Counter
|
||||
counts = Counter(v[0] for v in results.values())
|
||||
print("=== verdict histogram ===")
|
||||
for k, v in counts.most_common():
|
||||
print(f" {k}: {v}")
|
||||
print()
|
||||
print("=== everything that is NOT byte-identical ===")
|
||||
for key in sorted(results):
|
||||
verdict, detail = results[key]
|
||||
if verdict not in ("IDENTICAL", "IDENTICAL@PINNED", "FIRST-PARTY-OK"):
|
||||
print(f" {key}: {verdict} [{detail}]")
|
||||
json.dump({k: list(v) for k, v in results.items()},
|
||||
open("/tmp/audit_licenses_results.json", "w"), indent=1)
|
||||
print("\nfull results: /tmp/audit_licenses_results.json")
|
||||
|
||||
accepted = {"IDENTICAL", "IDENTICAL@PINNED", "FIRST-PARTY-OK",
|
||||
"FIRST-PARTY-NOTICE", "MANUAL-VERIFIED"}
|
||||
bad = {k: v for k, v in results.items() if v[0] not in accepted}
|
||||
if bad:
|
||||
print(f"\nPROVENANCE AUDIT FAILED: {len(bad)} unexplained verdict(s)")
|
||||
sys.exit(1)
|
||||
print("\nPROVENANCE AUDIT PASSED: every vendored license accounted for")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Index a single benchmark repository and capture metrics.
|
||||
# Usage: benchmark-index.sh <binary> <lang> <repo_path> <results_dir>
|
||||
|
||||
BINARY="${1:?Usage: benchmark-index.sh <binary> <lang> <repo_path> <results_dir>}"
|
||||
LANG="${2:?}"
|
||||
REPO="${3:?}"
|
||||
RESULTS_DIR="${4:?}"
|
||||
|
||||
# Resolve symlinks
|
||||
REPO=$(cd "$REPO" && pwd -P)
|
||||
|
||||
OUT="$RESULTS_DIR/$LANG"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
echo "INDEX: $LANG ($REPO)"
|
||||
|
||||
# Count source files and LOC (exclude .git, vendor, node_modules, build dirs)
|
||||
FILE_COUNT=$(find "$REPO" -type f \
|
||||
! -path '*/.git/*' ! -path '*/node_modules/*' ! -path '*/vendor/*' \
|
||||
! -path '*/target/*' ! -path '*/build/*' ! -path '*/dist/*' \
|
||||
! -path '*/__pycache__/*' ! -path '*/.cache/*' \
|
||||
| wc -l | tr -d ' ')
|
||||
|
||||
LOC=$(find "$REPO" -type f \
|
||||
! -path '*/.git/*' ! -path '*/node_modules/*' ! -path '*/vendor/*' \
|
||||
! -path '*/target/*' ! -path '*/build/*' ! -path '*/dist/*' \
|
||||
! -path '*/__pycache__/*' ! -path '*/.cache/*' \
|
||||
-exec cat {} + 2>/dev/null | wc -l | tr -d ' ')
|
||||
|
||||
echo "$FILE_COUNT" > "$OUT/file-count.txt"
|
||||
echo "$LOC" > "$OUT/loc.txt"
|
||||
|
||||
# Index via CLI and capture timing
|
||||
START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
|
||||
|
||||
INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}')
|
||||
|
||||
END_MS=$(python3 -c "import time; print(int(time.time()*1000))")
|
||||
ELAPSED=$((END_MS - START_MS))
|
||||
|
||||
echo "$INDEX_JSON" > "$OUT/00-index.json"
|
||||
echo "$ELAPSED" > "$OUT/index-time.txt"
|
||||
|
||||
# Extract node/edge counts (CLI wraps in MCP content envelope)
|
||||
NODES=$(echo "$INDEX_JSON" | python3 -c "
|
||||
import json,sys
|
||||
d=json.load(sys.stdin)
|
||||
# Unwrap MCP content envelope if present
|
||||
if 'content' in d:
|
||||
inner=json.loads(d['content'][0]['text'])
|
||||
else:
|
||||
inner=d
|
||||
print(inner.get('nodes',0))
|
||||
" 2>/dev/null || echo "0")
|
||||
EDGES=$(echo "$INDEX_JSON" | python3 -c "
|
||||
import json,sys
|
||||
d=json.load(sys.stdin)
|
||||
if 'content' in d:
|
||||
inner=json.loads(d['content'][0]['text'])
|
||||
else:
|
||||
inner=d
|
||||
print(inner.get('edges',0))
|
||||
" 2>/dev/null || echo "0")
|
||||
PROJECT=$(echo "$INDEX_JSON" | python3 -c "
|
||||
import json,sys
|
||||
d=json.load(sys.stdin)
|
||||
if 'content' in d:
|
||||
inner=json.loads(d['content'][0]['text'])
|
||||
else:
|
||||
inner=d
|
||||
print(inner.get('project',''))
|
||||
" 2>/dev/null || echo "")
|
||||
|
||||
echo "$NODES" > "$OUT/nodes.txt"
|
||||
echo "$EDGES" > "$OUT/edges.txt"
|
||||
echo "$PROJECT" > "$OUT/project.txt"
|
||||
|
||||
printf " %s: %s files, %s LOC, %sms, %s nodes, %s edges\n" \
|
||||
"$LANG" "$FILE_COUNT" "$LOC" "$ELAPSED" "$NODES" "$EDGES"
|
||||
Executable
+63
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env bash
|
||||
# benchmark-search-graph.sh — Time search_graph name_pattern= queries against a
|
||||
# codebase-memory-mcp binary to measure the regex / LIKE pre-filter performance.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/benchmark-search-graph.sh <binary-path> <project-name>
|
||||
#
|
||||
# Example:
|
||||
# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp my-project
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BINARY="${1:?Usage: $0 <binary-path> <project-name>}"
|
||||
PROJECT="${2:?Usage: $0 <binary-path> <project-name>}"
|
||||
|
||||
echo "Binary: $BINARY"
|
||||
echo "Project: $PROJECT"
|
||||
echo ""
|
||||
|
||||
run_case() {
|
||||
local label="$1"
|
||||
local request="$2"
|
||||
local start end elapsed_ms result
|
||||
|
||||
start=$(date +%s%3N)
|
||||
result=$(echo "$request" | "$BINARY" 2>/dev/null || true)
|
||||
end=$(date +%s%3N)
|
||||
elapsed_ms=$(( end - start ))
|
||||
|
||||
local count
|
||||
count=$(echo "$result" | python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
d = json.load(sys.stdin)
|
||||
content = d.get('result', {}).get('content', [{}])[0].get('text', '{}')
|
||||
obj = json.loads(content)
|
||||
print(obj.get('total', obj.get('count', '?')))
|
||||
except Exception:
|
||||
print('?')
|
||||
" 2>/dev/null || echo "?")
|
||||
|
||||
printf " %-55s %5dms (total=%s)\n" "$label" "$elapsed_ms" "$count"
|
||||
}
|
||||
|
||||
sg() {
|
||||
local project="$1"
|
||||
local args="$2"
|
||||
printf '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"search_graph","arguments":{"project":"%s",%s}}}' \
|
||||
"$project" "$args"
|
||||
}
|
||||
|
||||
echo "=== search_graph name_pattern= benchmarks ==="
|
||||
run_case "name_pattern=.*Controller.*" "$(sg "$PROJECT" '"name_pattern":".*Controller.*","limit":20')"
|
||||
run_case "name_pattern=.*Service.*" "$(sg "$PROJECT" '"name_pattern":".*Service.*","limit":20')"
|
||||
run_case "name_pattern=.*Repository.*" "$(sg "$PROJECT" '"name_pattern":".*Repository.*","limit":20')"
|
||||
run_case "name_pattern=specificFunctionName" "$(sg "$PROJECT" '"name_pattern":"specificFunctionName","limit":20')"
|
||||
run_case "label=Method + name_pattern=.*get.*" "$(sg "$PROJECT" '"label":"Method","name_pattern":".*get.*","limit":20')"
|
||||
|
||||
echo ""
|
||||
echo "=== search_graph query= benchmarks (BM25 path) ==="
|
||||
run_case "query=controller service handler" "$(sg "$PROJECT" '"query":"controller service handler","limit":20')"
|
||||
run_case "query=user authentication permission role" "$(sg "$PROJECT" '"query":"user authentication permission role","limit":20')"
|
||||
run_case "query=create update delete manage list view admin" "$(sg "$PROJECT" '"query":"create update delete manage list view admin","limit":20')"
|
||||
Executable
+99
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env bash
|
||||
# build.sh — Clean build of production binary (standard or with UI).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/build.sh # Standard binary
|
||||
# scripts/build.sh --with-ui # Binary with embedded UI
|
||||
# scripts/build.sh --version v0.8.0 # With version stamp
|
||||
# scripts/build.sh --arch x86_64 # Force x86_64 build
|
||||
# scripts/build.sh CC=gcc-14 CXX=g++-14 # Override compiler
|
||||
#
|
||||
# This script is the SINGLE source of truth for building release binaries.
|
||||
# Used identically in local development and CI workflows.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Pre-parse --arch flag before sourcing env.sh
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--arch=*) export CBM_ARCH="${arg#--arch=}" ;;
|
||||
esac
|
||||
done
|
||||
prev_arg=""
|
||||
for arg in "$@"; do
|
||||
if [[ "${prev_arg:-}" == "--arch" ]]; then
|
||||
export CBM_ARCH="$arg"
|
||||
fi
|
||||
prev_arg="$arg"
|
||||
done
|
||||
|
||||
# shellcheck source=env.sh
|
||||
source "$ROOT/scripts/env.sh"
|
||||
|
||||
# Parse remaining arguments
|
||||
WITH_UI=false
|
||||
VERSION=""
|
||||
EXTRA_MAKE_ARGS=()
|
||||
|
||||
prev_arg=""
|
||||
for arg in "$@"; do
|
||||
# Skip --arch and its value (already handled)
|
||||
if [[ "${prev_arg:-}" == "--arch" ]]; then
|
||||
prev_arg="$arg"
|
||||
continue
|
||||
fi
|
||||
case "$arg" in
|
||||
--with-ui)
|
||||
WITH_UI=true
|
||||
;;
|
||||
--version)
|
||||
prev_arg="$arg"
|
||||
continue
|
||||
;;
|
||||
--arch|--arch=*)
|
||||
;; # already handled
|
||||
CC=*|CXX=*)
|
||||
export "${arg}"
|
||||
EXTRA_MAKE_ARGS+=("$arg")
|
||||
;;
|
||||
*)
|
||||
# Check if this is the value for --version
|
||||
if [[ "${prev_arg:-}" == "--version" ]]; then
|
||||
VERSION="$arg"
|
||||
else
|
||||
EXTRA_MAKE_ARGS+=("$arg")
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
prev_arg="$arg"
|
||||
done
|
||||
|
||||
# Version flag
|
||||
CFLAGS_EXTRA=""
|
||||
if [[ -n "$VERSION" ]]; then
|
||||
CLEAN_VERSION="${VERSION#v}"
|
||||
CFLAGS_EXTRA="-DCBM_VERSION=\"\\\"$CLEAN_VERSION\\\"\""
|
||||
fi
|
||||
|
||||
print_env "build.sh"
|
||||
echo " ui=$WITH_UI version=${VERSION:-dev}"
|
||||
|
||||
# Verify compiler supports target arch
|
||||
verify_compiler "$CC"
|
||||
|
||||
# Step 1: Clean C build artifacts only (not node_modules — npm ci handles that)
|
||||
rm -rf "$ROOT/build/c"
|
||||
|
||||
# Step 2: Build (Makefile applies $ARCHFLAGS for the target arch on macOS)
|
||||
if $WITH_UI; then
|
||||
make -j"$NPROC" -f Makefile.cbm cbm-with-ui \
|
||||
CFLAGS_EXTRA="$CFLAGS_EXTRA" "${EXTRA_MAKE_ARGS[@]+"${EXTRA_MAKE_ARGS[@]}"}"
|
||||
else
|
||||
make -j"$NPROC" -f Makefile.cbm cbm \
|
||||
CFLAGS_EXTRA="$CFLAGS_EXTRA" "${EXTRA_MAKE_ARGS[@]+"${EXTRA_MAKE_ARGS[@]}"}"
|
||||
fi
|
||||
|
||||
echo "=== Build complete: build/c/codebase-memory-mcp ==="
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# DCO enforcement: every commit in the given range must carry a
|
||||
# Signed-off-by trailer whose email matches the commit author
|
||||
# (Developer Certificate of Origin 1.1 — see the DCO file).
|
||||
#
|
||||
# Usage: check-dco.sh <range> e.g. origin/main..HEAD, sha1..sha2
|
||||
#
|
||||
# Exemptions (same as the standard DCO checks): merge commits and
|
||||
# bot-authored commits (author name ending in [bot]).
|
||||
|
||||
RANGE="${1:?usage: check-dco.sh <commit-range>}"
|
||||
|
||||
FAIL=0
|
||||
CHECKED=0
|
||||
while IFS= read -r sha; do
|
||||
# Skip merge commits
|
||||
nparents=$(git rev-list --no-walk --parents -n1 "$sha" | wc -w)
|
||||
if [ "$nparents" -gt 2 ]; then
|
||||
continue
|
||||
fi
|
||||
author_name=$(git log -1 --format='%an' "$sha")
|
||||
case "$author_name" in
|
||||
*"[bot]") continue ;;
|
||||
esac
|
||||
CHECKED=$((CHECKED + 1))
|
||||
author_email=$(git log -1 --format='%ae' "$sha")
|
||||
trailers=$(git log -1 --format='%(trailers:key=Signed-off-by,valueonly)' "$sha")
|
||||
if ! printf '%s' "$trailers" | grep -qiF "<$author_email>"; then
|
||||
echo "BLOCKED: $sha lacks a Signed-off-by matching its author:"
|
||||
git log -1 --format=' author: %an <%ae>%n subject: %s' "$sha"
|
||||
echo " fix: git commit --amend -s (or: git rebase --signoff <base>)"
|
||||
FAIL=1
|
||||
fi
|
||||
done < <(git rev-list "$RANGE")
|
||||
|
||||
if [ "$FAIL" -ne 0 ]; then
|
||||
echo "=== DCO CHECK FAILED — every commit must be signed off (git commit -s) ==="
|
||||
echo "=== See the DCO file and CONTRIBUTING.md ==="
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: $CHECKED commit(s) in $RANGE carry a valid Signed-off-by"
|
||||
Executable
+227
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env bash
|
||||
# check-lsp-originality.sh — Provenance guard for the hybrid LSP implementations.
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
# The LSP layer (internal/cbm/lsp/) is an original C implementation whose
|
||||
# behavior is "structurally inspired by and compatible with" the reference
|
||||
# language servers (pyright, gopls, tsserver, Roslyn, clangd, rust-analyzer,
|
||||
# …). We claim it contains NO source copied from those projects. This script
|
||||
# is the documented, runnable check that backs that claim — run it before
|
||||
# committing any NEW LSP implementation work, and review its report.
|
||||
#
|
||||
# WHAT IT CATCHES (the realistic copy "tells", and they survive a language port)
|
||||
# 1. Verbatim string / error-message literals shared with a reference.
|
||||
# 2. Verbatim comment phrases shared with a reference (the classic port tell:
|
||||
# people re-type the logic but keep the explanatory comments).
|
||||
# 3. Same-language structural clones via jscpd (uses `jscpd` on PATH, else
|
||||
# `npx --yes jscpd`) — token-level clone detection, run only for clangd
|
||||
# (C++ ↔ our C), the one reference close enough to C to tokenize alike.
|
||||
#
|
||||
# WHAT IT CANNOT CATCH (be honest — these need a human)
|
||||
# An algorithm ported line-by-line with every identifier/comment rewritten is
|
||||
# NOT machine-detectable cheaply across C↔TS/Go/C#/Rust/Java. A clean report
|
||||
# is necessary, not sufficient: it means "no verbatim overlap found", not
|
||||
# "provably independent". The git history (incremental, test-driven authorship)
|
||||
# is the complementary evidence; keep it clean.
|
||||
#
|
||||
# USAGE
|
||||
# bash scripts/check-lsp-originality.sh [--lang NAME] [--refresh]
|
||||
# [--list-candidates] [--help]
|
||||
# --lang NAME scan against ONE reference only (py|ts|go|cs|c|java|kotlin|rust|php|perl)
|
||||
# --refresh re-fetch reference sources even if cached
|
||||
# --list-candidates print the extracted local tokens and exit (no fetch; self-test)
|
||||
# Exit 0 = no verbatim overlap found. Exit 1 = overlap(s) to review by a human.
|
||||
#
|
||||
# Reference sources are shallow/sparse-cloned into $LSP_REFS_DIR (default
|
||||
# .lsp-refs/, gitignored). Adding a new LSP language? Add its upstream repo to
|
||||
# the REFS manifest below so this guard covers it too.
|
||||
set -uo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
TAG="[lsp-orig]"
|
||||
LSP_DIR="$ROOT/internal/cbm/lsp"
|
||||
REFS_DIR="${LSP_REFS_DIR:-$ROOT/.lsp-refs}"
|
||||
|
||||
# Tunables (override via env).
|
||||
MIN_STR="${LSP_ORIG_MIN_STR:-16}" # min length of a string literal to consider
|
||||
MIN_COMMENT="${LSP_ORIG_MIN_COMMENT:-30}" # min length of a comment phrase to consider
|
||||
MIN_TOKENS="${LSP_ORIG_MIN_TOKENS:-50}" # jscpd structural-clone min token run
|
||||
|
||||
# Reference manifest: lang | git URL | sparse subpath (light checkout)
|
||||
REFS=(
|
||||
"py|https://github.com/microsoft/pyright|packages/pyright-internal/src"
|
||||
"ts|https://github.com/microsoft/TypeScript|src/compiler,src/services"
|
||||
"go|https://github.com/golang/tools|gopls/internal"
|
||||
"cs|https://github.com/dotnet/roslyn|src/Compilers/CSharp/Portable"
|
||||
"c|https://github.com/llvm/llvm-project|clang-tools-extra/clangd"
|
||||
"java|https://github.com/eclipse-jdtls/eclipse.jdt.ls|org.eclipse.jdt.ls.core/src"
|
||||
"kotlin|https://github.com/fwcd/kotlin-language-server|server/src"
|
||||
"rust|https://github.com/rust-lang/rust-analyzer|crates"
|
||||
# PHP was implemented from specs (PHP language ref + PSR-4), with no upstream
|
||||
# LSP. We scan against phpactor (MIT) — the leading OSS PHP language server —
|
||||
# purely for defensive copy-detection. NOT Intelephense (proprietary).
|
||||
"php|https://github.com/phpactor/phpactor|lib"
|
||||
# Perl (PR #461) was authored clean-room. We scan against PerlNavigator (MIT)
|
||||
# — the leading OSS Perl language server — for defensive copy-detection.
|
||||
"perl|https://github.com/bscan/PerlNavigator|server/src"
|
||||
)
|
||||
|
||||
ONLY_LANG=""; REFRESH=0; LIST_ONLY=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--lang) ONLY_LANG="${2:-}"; shift 2 ;;
|
||||
--refresh) REFRESH=1; shift ;;
|
||||
--list-candidates) LIST_ONLY=1; shift ;;
|
||||
--help|-h) sed -n '2,40p' "$0"; exit 0 ;;
|
||||
*) echo "$TAG unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if ! command -v rg >/dev/null 2>&1; then
|
||||
echo "$TAG ERROR: ripgrep (rg) is required." >&2; exit 2
|
||||
fi
|
||||
|
||||
# --- 1. Extract distinctive local tokens from our LSP C sources ---------------
|
||||
# Skip generated/ (machine-emitted stdlib tables, not hand-authored logic).
|
||||
# bash 3.2 (macOS default) has no `mapfile` — build the array with a read loop.
|
||||
LSP_SRC=()
|
||||
while IFS= read -r f; do LSP_SRC+=("$f"); done < <(
|
||||
find "$LSP_DIR" -type f \( -name '*.c' -o -name '*.h' \) -not -path '*/generated/*' | sort)
|
||||
if [ "${#LSP_SRC[@]}" -eq 0 ]; then
|
||||
echo "$TAG ERROR: no LSP sources under $LSP_DIR" >&2; exit 2
|
||||
fi
|
||||
|
||||
CAND="$(mktemp)"; trap 'rm -f "$CAND" "$CAND.s" "$CAND.c" 2>/dev/null' EXIT
|
||||
|
||||
# String literals of interest: >= MIN_STR chars, contain a letter and a space
|
||||
# (favours human-readable phrases / messages over format specifiers & symbols).
|
||||
rg --no-filename -o "\"[^\"]{${MIN_STR},}\"" "${LSP_SRC[@]}" 2>/dev/null \
|
||||
| sed -E 's/^"//; s/"$//' \
|
||||
| rg '[A-Za-z].* ' \
|
||||
| rg -v '^(%|https?://|[0-9. ]+$)' \
|
||||
| sort -u > "$CAND.s"
|
||||
|
||||
# Comment phrases: // line comments and single-line /* ... */, >= MIN_COMMENT.
|
||||
{ rg --no-filename -o '//[^\n]+' "${LSP_SRC[@]}" 2>/dev/null | sed -E 's#^//+##'
|
||||
rg --no-filename -o '/\*.*\*/' "${LSP_SRC[@]}" 2>/dev/null | sed -E 's#^/\*+##; s#\*+/$##'
|
||||
} | sed -E 's/^[[:space:]*]+//; s/[[:space:]]+$//' \
|
||||
| awk -v n="$MIN_COMMENT" 'length($0) >= n' \
|
||||
| rg -v -i '^(todo|fixme|note|hack|xxx|copyright|spdx|clang-format|nolint|fallthrough)\b' \
|
||||
| sort -u > "$CAND.c"
|
||||
|
||||
# Keep only "meaningful" tokens: at least two real words (drops decorative
|
||||
# dividers like ===== / ----- / box-drawing rules that collide with banners).
|
||||
cat "$CAND.s" "$CAND.c" | grep -v '^[[:space:]]*$' \
|
||||
| rg -P '[A-Za-z]{3,}[^A-Za-z]+[A-Za-z]{2,}' \
|
||||
| sort -u > "$CAND"
|
||||
n_str=$(wc -l < "$CAND.s" | tr -d ' '); n_com=$(wc -l < "$CAND.c" | tr -d ' ')
|
||||
echo "$TAG extracted $(wc -l < "$CAND" | tr -d ' ') candidate tokens from ${#LSP_SRC[@]} LSP sources ($n_str strings, $n_com comments)"
|
||||
|
||||
if [ "$LIST_ONLY" -eq 1 ]; then
|
||||
echo "$TAG --- candidate tokens (no fetch performed) ---"
|
||||
cat "$CAND"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -s "$CAND" ]; then
|
||||
echo "$TAG no candidate tokens — nothing to compare."; exit 0
|
||||
fi
|
||||
|
||||
# --- 2. Fetch reference sources (shallow + sparse) ----------------------------
|
||||
fetch_ref() {
|
||||
local lang="$1" url="$2" subpath="$3" dst="$REFS_DIR/$lang"
|
||||
if [ -d "$dst/.git" ] && [ "$REFRESH" -eq 0 ]; then return 0; fi
|
||||
rm -rf "$dst"; mkdir -p "$dst"
|
||||
echo "$TAG fetching $lang ($url :: $subpath) ..."
|
||||
if ! git clone --depth 1 --filter=blob:none --sparse "$url" "$dst" >/dev/null 2>&1; then
|
||||
echo "$TAG WARN: clone failed for $lang — skipping"; rm -rf "$dst"; return 1
|
||||
fi
|
||||
( cd "$dst" && git sparse-checkout set ${subpath//,/ } >/dev/null 2>&1 ) || true
|
||||
}
|
||||
|
||||
# --- 3. Search candidates in each reference -----------------------------------
|
||||
# Restrict to source code — license texts, changelogs and docs are not "copying".
|
||||
REF_GLOBS=(-g '!**/LICENSE*' -g '!**/COPYING*' -g '!**/NOTICE*' -g '!**/CHANGELOG*'
|
||||
-g '!**/*.md' -g '!**/*.txt' -g '!**/*.json' -g '!**/*.yml' -g '!**/*.yaml')
|
||||
total_hits=0
|
||||
for entry in "${REFS[@]}"; do
|
||||
IFS='|' read -r lang url subpath <<< "$entry"
|
||||
[ -n "$ONLY_LANG" ] && [ "$ONLY_LANG" != "$lang" ] && continue
|
||||
fetch_ref "$lang" "$url" "$subpath" || continue
|
||||
dst="$REFS_DIR/$lang"
|
||||
[ -d "$dst" ] || continue
|
||||
|
||||
# Fixed-string search of every candidate across this reference tree.
|
||||
hits=$(rg -F -l "${REF_GLOBS[@]}" -f "$CAND" "$dst" 2>/dev/null | wc -l | tr -d ' ')
|
||||
if [ "${hits:-0}" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$TAG ⚠ overlap with reference '$lang' ($url):"
|
||||
# Show which candidate matched where (cap output).
|
||||
rg -F -n "${REF_GLOBS[@]}" -f "$CAND" "$dst" 2>/dev/null | head -40
|
||||
total_hits=$((total_hits + hits))
|
||||
else
|
||||
echo "$TAG ok — no verbatim overlap with '$lang'"
|
||||
fi
|
||||
done
|
||||
|
||||
# --- 4. Structural-clone pass (clangd C++ <-> our C; the only same-ish language)
|
||||
# String search misses ports that kept the code structure but rewrote identifiers
|
||||
# and strings. jscpd is a token clone detector; it tokenizes per format, so we
|
||||
# stage both trees as .cpp (C tokenizes fine as C++) and report only clone pairs
|
||||
# that SPAN the two trees.
|
||||
structural_hits=0
|
||||
if [ -z "$ONLY_LANG" ] || [ "$ONLY_LANG" = "c" ]; then
|
||||
JSCPD=""
|
||||
if command -v jscpd >/dev/null 2>&1; then JSCPD="jscpd"
|
||||
elif command -v npx >/dev/null 2>&1; then JSCPD="npx --yes jscpd"; fi
|
||||
cdir="$REFS_DIR/c"
|
||||
if [ -z "$JSCPD" ]; then
|
||||
echo "$TAG structural pass skipped — install jscpd (npm i -g jscpd) or node/npx to enable"
|
||||
elif [ ! -d "$cdir" ]; then
|
||||
echo "$TAG structural pass skipped — clangd ref not fetched (run without --lang, or --lang c)"
|
||||
else
|
||||
stage="$(mktemp -d)"; jout="$(mktemp -d)"
|
||||
for f in "${LSP_SRC[@]}"; do
|
||||
cp "$f" "$stage/ours__$(echo "${f#$LSP_DIR/}" | tr '/' '_').cpp"
|
||||
done
|
||||
while IFS= read -r f; do
|
||||
cp "$f" "$stage/clangd__$(echo "${f#$cdir/}" | tr '/' '_').cpp" 2>/dev/null
|
||||
done < <(find "$cdir" -type f \( -name '*.cpp' -o -name '*.cc' -o -name '*.cxx' \
|
||||
-o -name '*.h' -o -name '*.hpp' \))
|
||||
echo "$TAG structural pass (jscpd, >= ${MIN_TOKENS} tokens, clangd C++ <-> our C) ..."
|
||||
$JSCPD --silent --reporters json --output "$jout" --min-tokens "$MIN_TOKENS" "$stage" >/dev/null 2>&1 || true
|
||||
pairs=""
|
||||
if [ -f "$jout/jscpd-report.json" ]; then
|
||||
pairs=$(jq -r '.duplicates[]?
|
||||
| select((.firstFile.name|contains("ours__")) != (.secondFile.name|contains("ours__")))
|
||||
| " \(.firstFile.name|gsub(".*/";"")) <-> \(.secondFile.name|gsub(".*/";"")) [\(.lines) lines]"' \
|
||||
"$jout/jscpd-report.json" 2>/dev/null | sort -u)
|
||||
fi
|
||||
if [ -n "$pairs" ]; then
|
||||
echo "$TAG ⚠ structural clones spanning our C and clangd:"; echo "$pairs" | head -20
|
||||
structural_hits=1
|
||||
else
|
||||
echo "$TAG ok — no structural clones (>= ${MIN_TOKENS} tokens) between our C and clangd"
|
||||
fi
|
||||
rm -rf "$stage" "$jout"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [ "$total_hits" -gt 0 ] || [ "$structural_hits" -gt 0 ]; then
|
||||
if [ "$total_hits" -gt 0 ]; then
|
||||
echo "$TAG REVIEW NEEDED: $total_hits reference file(s) share a verbatim string/comment"
|
||||
echo "$TAG with internal/cbm/lsp/. A hit is NOT proof of copying (common phrases collide)"
|
||||
echo "$TAG — inspect each, confirm independent wording, reword genuinely-copied text."
|
||||
fi
|
||||
if [ "$structural_hits" -gt 0 ]; then
|
||||
echo "$TAG REVIEW NEEDED: jscpd found structurally-cloned block(s) between our C and"
|
||||
echo "$TAG clangd — inspect the pairs above and confirm independent implementation."
|
||||
fi
|
||||
echo "$TAG Re-run until clean before committing."
|
||||
exit 1
|
||||
fi
|
||||
echo "$TAG CLEAN — no verbatim string/comment overlap with any reference, and no"
|
||||
echo "$TAG structural clones (jscpd) between our C and clangd."
|
||||
echo "$TAG (Reminder: this proves no verbatim/structural overlap, not provable"
|
||||
echo "$TAG independence — the incremental git history is the complementary evidence.)"
|
||||
exit 0
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# check-no-test-skips.sh — Enforce the no-skips test policy in the lint phase.
|
||||
#
|
||||
# Every test must PASS or FAIL. The ONLY tolerable skip is a genuinely
|
||||
# platform-specific test that cannot run on the current OS (e.g. a Windows-only
|
||||
# test on macOS); those must use the SKIP_PLATFORM() macro (or #ifdef
|
||||
# compile-gating). A plain SKIP() — or any direct tf_skip_count manipulation —
|
||||
# in a test source means a setup/environment/resource failure is being hidden
|
||||
# instead of surfaced as a failure. That fails this check.
|
||||
#
|
||||
# Rationale: a test that cannot establish its preconditions has FAILED, not been
|
||||
# "skipped". Convert such SKIP() to FAIL("reason").
|
||||
#
|
||||
# Usage: bash scripts/check-no-test-skips.sh (exit 0 = clean, 1 = violations)
|
||||
set -uo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
violations=0
|
||||
|
||||
# 1. Plain SKIP( in test sources.
|
||||
# - SKIP_PLATFORM( never matches "SKIP(" (the char after SKIP is '_').
|
||||
# - The SKIP()/FAIL()/SKIP_PLATFORM() macro DEFINITIONS live in
|
||||
# tests/test_framework.h (a .h), which the tests/*.c glob does not scan.
|
||||
while IFS= read -r hit; do
|
||||
echo "[no-skips] FORBIDDEN SKIP(): $hit"
|
||||
violations=$((violations + 1))
|
||||
done < <(grep -rnE '(^|[^A-Za-z0-9_])SKIP\(' "$ROOT"/tests/*.c 2>/dev/null || true)
|
||||
|
||||
# 2. Direct tf_skip_count increment in a test source (only the framework's
|
||||
# SKIP_PLATFORM macro, defined in the .h, may touch it).
|
||||
while IFS= read -r hit; do
|
||||
echo "[no-skips] FORBIDDEN tf_skip_count manipulation: $hit"
|
||||
violations=$((violations + 1))
|
||||
done < <(grep -rnE 'tf_skip_count[[:space:]]*(\+\+|\+=|--)' "$ROOT"/tests/*.c 2>/dev/null || true)
|
||||
|
||||
if [ "$violations" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "[no-skips] FAIL: $violations violation(s). Tests must pass or fail."
|
||||
echo " setup / environment / resource failures -> FAIL(\"reason\")"
|
||||
echo " genuinely platform-specific tests -> SKIP_PLATFORM(\"reason\") or #ifdef"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[no-skips] OK — no forbidden skips in tests/*.c (SKIP_PLATFORM allowed for platform-only tests)"
|
||||
exit 0
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# Verify NOLINT(misc-no-recursion) only appears on whitelisted functions.
|
||||
WHITELIST="src/foundation/recursion_whitelist.h"
|
||||
if [ ! -f "$WHITELIST" ]; then
|
||||
echo "ERROR: $WHITELIST not found"; exit 1
|
||||
fi
|
||||
ALLOWED=$(grep -oE '"[a-zA-Z_][a-zA-Z0-9_]*"' "$WHITELIST" | tr -d '"' | sort -u)
|
||||
|
||||
HITS=$(grep -rn 'NOLINT(misc-no-recursion)' src/ internal/cbm/*.c internal/cbm/*.h 2>/dev/null \
|
||||
| grep -v vendored | grep -v recursion_whitelist)
|
||||
if [ -z "$HITS" ]; then exit 0; fi
|
||||
|
||||
FAIL=0
|
||||
while IFS= read -r line; do
|
||||
FILE=$(echo "$line" | cut -d: -f1)
|
||||
LN=$(echo "$line" | cut -d: -f2)
|
||||
# Check current line AND 5 lines above for the function name
|
||||
CONTEXT=$(sed -n "$((LN > 5 ? LN - 5 : 1)),${LN}p" "$FILE")
|
||||
FOUND=0
|
||||
for fn in $ALLOWED; do
|
||||
if echo "$CONTEXT" | grep -qw "$fn"; then
|
||||
FOUND=1; break
|
||||
fi
|
||||
done
|
||||
if [ "$FOUND" -eq 0 ]; then
|
||||
echo "ERROR: NOLINT(misc-no-recursion) on non-whitelisted function:"
|
||||
echo " $line"
|
||||
FAIL=1
|
||||
fi
|
||||
done <<< "$HITS"
|
||||
exit $FAIL
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regression guard: the linux binary we ship must START on OLD glibc.
|
||||
#
|
||||
# The standard linux release binary dynamically links glibc 2.38+ / GLIBCXX
|
||||
# 3.4.32 and fails to start on Debian 11, RHEL/Rocky 8, Ubuntu 20.04, Amazon
|
||||
# Linux 2, etc. The fix points all linux install + self-update paths at the
|
||||
# fully-static "-portable" asset. This runs a given linux binary inside an
|
||||
# old-glibc container (debian:bullseye, glibc 2.31) and asserts it starts:
|
||||
# - RED for the dynamic standard binary (GLIBC_2.38 not found)
|
||||
# - GREEN for the static -portable binary (runs anywhere)
|
||||
#
|
||||
# Usage: check-glibc-compat.sh <path-to-linux-binary>
|
||||
# Env: GLIBC_TEST_IMAGE (default: debian:bullseye-slim, glibc 2.31)
|
||||
set -euo pipefail
|
||||
|
||||
BIN="${1:?usage: check-glibc-compat.sh <path-to-linux-binary>}"
|
||||
IMAGE="${GLIBC_TEST_IMAGE:-debian:bullseye-slim}"
|
||||
BIN_ABS="$(cd "$(dirname "$BIN")" && pwd)/$(basename "$BIN")"
|
||||
|
||||
echo "==> running $(basename "$BIN") --version inside ${IMAGE} (glibc 2.31)"
|
||||
docker run --rm -v "${BIN_ABS}:/cbm:ro" "${IMAGE}" /cbm --version
|
||||
echo "PASS: binary starts on old glibc (${IMAGE})"
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
# Wait for VirusTotal scans to complete and check results.
|
||||
# Expects: VT_API_KEY, VT_ANALYSIS (comma-separated "file=URL" pairs)
|
||||
set -euo pipefail
|
||||
|
||||
MIN_ENGINES=60
|
||||
rm -f /tmp/vt_gate_fail
|
||||
|
||||
echo "=== Waiting for VirusTotal scans to fully complete ==="
|
||||
|
||||
echo "$VT_ANALYSIS" | tr ',' '\n' | while IFS= read -r entry; do
|
||||
[ -z "$entry" ] && continue
|
||||
FILE=$(echo "$entry" | cut -d'=' -f1)
|
||||
URL=$(echo "$entry" | cut -d'=' -f2-)
|
||||
BASENAME=$(basename "$FILE")
|
||||
|
||||
# Extract base64 analysis ID from URL
|
||||
ANALYSIS_ID=$(echo "$URL" | sed -n 's|.*/file-analysis/\([^/]*\)/.*|\1|p')
|
||||
if [ -z "$ANALYSIS_ID" ]; then
|
||||
ANALYSIS_ID=$(echo "$URL" | grep -oE '[a-f0-9]{64}')
|
||||
if [ -z "$ANALYSIS_ID" ]; then
|
||||
echo "BLOCKED: Cannot parse VirusTotal URL: $URL"
|
||||
echo "FAIL" >> /tmp/vt_gate_fail
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Poll until completed (max 120 min)
|
||||
SCAN_COMPLETE=false
|
||||
for attempt in $(seq 1 720); do
|
||||
RESULT=$(curl -sf --max-time 10 \
|
||||
-H "x-apikey: $VT_API_KEY" \
|
||||
"https://www.virustotal.com/api/v3/analyses/$ANALYSIS_ID" 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$RESULT" ]; then
|
||||
echo " $BASENAME: waiting (attempt $attempt)..."
|
||||
sleep 10
|
||||
continue
|
||||
fi
|
||||
|
||||
STATS=$(echo "$RESULT" | python3 -c "
|
||||
import json, sys
|
||||
d = json.loads(sys.stdin.read())
|
||||
attrs = d.get('data', {}).get('attributes', {})
|
||||
status = attrs.get('status', 'queued')
|
||||
stats = attrs.get('stats', {})
|
||||
malicious = stats.get('malicious', 0)
|
||||
suspicious = stats.get('suspicious', 0)
|
||||
undetected = stats.get('undetected', 0)
|
||||
harmless = stats.get('harmless', 0)
|
||||
total = sum(stats.values())
|
||||
completed = malicious + suspicious + undetected + harmless
|
||||
print(f'{status},{malicious},{suspicious},{completed},{total}')
|
||||
" 2>/dev/null || echo "queued,0,0,0,0")
|
||||
|
||||
STATUS=$(echo "$STATS" | cut -d',' -f1)
|
||||
MALICIOUS=$(echo "$STATS" | cut -d',' -f2)
|
||||
SUSPICIOUS=$(echo "$STATS" | cut -d',' -f3)
|
||||
COMPLETED=$(echo "$STATS" | cut -d',' -f4)
|
||||
TOTAL=$(echo "$STATS" | cut -d',' -f5)
|
||||
|
||||
if [ "$STATUS" = "completed" ]; then
|
||||
SCAN_COMPLETE=true
|
||||
if [ "$COMPLETED" -lt "$MIN_ENGINES" ]; then
|
||||
echo "NOTE: $BASENAME completed with only $COMPLETED/$TOTAL engines (< $MIN_ENGINES)"
|
||||
fi
|
||||
if [ "$MALICIOUS" -gt 0 ] || [ "$SUSPICIOUS" -gt 0 ]; then
|
||||
echo "BLOCKED: $BASENAME flagged ($MALICIOUS malicious, $SUSPICIOUS suspicious / $COMPLETED engines)"
|
||||
echo " $URL"
|
||||
echo "FAIL" >> /tmp/vt_gate_fail
|
||||
else
|
||||
echo "OK: $BASENAME clean ($COMPLETED engines, 0 detections)"
|
||||
fi
|
||||
break
|
||||
fi
|
||||
|
||||
echo " $BASENAME: $COMPLETED/$TOTAL engines ($STATUS, attempt $attempt)..."
|
||||
sleep 10
|
||||
done
|
||||
|
||||
if [ "$SCAN_COMPLETE" != "true" ]; then
|
||||
echo "WARNING: $BASENAME scan did not complete in time"
|
||||
echo "FAIL" >> /tmp/vt_gate_fail
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -f /tmp/vt_gate_fail ]; then
|
||||
echo "BLOCKED: One or more VirusTotal checks failed"
|
||||
exit 1
|
||||
fi
|
||||
echo "=== All VirusTotal scans passed ==="
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# clean.sh — Remove ALL build artifacts, caches, and generated files.
|
||||
#
|
||||
# Usage: scripts/clean.sh
|
||||
#
|
||||
# Ensures every subsequent build starts from scratch — no cached .o files,
|
||||
# no stale node_modules, no leftover dist folders. This is the first step
|
||||
# in both scripts/test.sh and scripts/build.sh.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
echo "=== Cleaning build artifacts ==="
|
||||
|
||||
# C build artifacts
|
||||
rm -rf "$ROOT/build/c"
|
||||
|
||||
# Frontend build artifacts
|
||||
rm -rf "$ROOT/graph-ui/dist"
|
||||
rm -rf "$ROOT/graph-ui/node_modules"
|
||||
|
||||
# Root-level node artifacts (if any)
|
||||
rm -rf "$ROOT/node_modules"
|
||||
|
||||
# Generated embedded assets (regenerated by embed-frontend.sh)
|
||||
rm -f "$ROOT/src/ui/embedded_assets.c"
|
||||
|
||||
# Leftover test fixture dirs (C test suite sometimes creates these in CWD)
|
||||
find "$ROOT" -maxdepth 1 -type d \( -name 'cbm_*' -o -name 'cli-*' \) -exec rm -rf {} + 2>/dev/null || true
|
||||
|
||||
# Leftover test fixture dirs in /tmp
|
||||
find /tmp -maxdepth 1 -type d \( -name 'cbm_*' -o -name 'cli-*' \) -user "$(id -u)" -exec rm -rf {} + 2>/dev/null || true
|
||||
|
||||
echo "=== Clean complete ==="
|
||||
Executable
+110
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Clone benchmark repositories for MCP vs Explorer quality comparison.
|
||||
# Uses shallow clones (--depth 1) to minimize disk usage.
|
||||
# Shared repos are cloned once and symlinked for secondary languages.
|
||||
|
||||
BENCH_DIR="${1:-/tmp/bench}"
|
||||
|
||||
clone() {
|
||||
local lang="$1" repo="$2" subdir="${3:-}"
|
||||
local dest="$BENCH_DIR/$lang"
|
||||
if [ -d "$dest" ]; then
|
||||
echo "SKIP: $lang (exists)"
|
||||
return
|
||||
fi
|
||||
echo "CLONE: $lang <- $repo"
|
||||
git clone --depth 1 --quiet "https://github.com/$repo.git" "$dest"
|
||||
echo " OK: $(du -sh "$dest" | cut -f1)"
|
||||
}
|
||||
|
||||
symlink() {
|
||||
local lang="$1" source_lang="$2"
|
||||
local dest="$BENCH_DIR/$lang"
|
||||
if [ -d "$dest" ] || [ -L "$dest" ]; then
|
||||
echo "SKIP: $lang (exists)"
|
||||
return
|
||||
fi
|
||||
echo "LINK: $lang -> $source_lang"
|
||||
ln -s "$BENCH_DIR/$source_lang" "$dest"
|
||||
}
|
||||
|
||||
mkdir -p "$BENCH_DIR"
|
||||
|
||||
# Programming languages — Tier 1 (44 languages)
|
||||
# Target: 100K+ LOC per repo for meaningful performance benchmarks
|
||||
clone go "kubernetes/kubernetes" # 3.5M LOC, the Go benchmark
|
||||
clone python "django/django" # 350K+ LOC, web framework
|
||||
clone javascript "vercel/next.js" # 500K+ LOC, React framework
|
||||
clone typescript "microsoft/TypeScript" # 1M+ LOC, the TS compiler
|
||||
clone tsx "shadcn-ui/ui" # 728K LOC (already large)
|
||||
clone java "elastic/elasticsearch" # 2M+ LOC, search engine
|
||||
clone kotlin "JetBrains/Exposed" # 977K LOC (already large)
|
||||
clone scala "apache/spark" # 1M+ LOC, big data
|
||||
clone rust "meilisearch/meilisearch" # 409K LOC (already large)
|
||||
clone c "redis/redis" # 546K LOC (already large)
|
||||
clone cpp "protocolbuffers/protobuf" # 500K+ LOC, real .cpp files
|
||||
clone csharp "dotnet/runtime" # Massive C# runtime
|
||||
clone php "koel/koel" # 189K LOC (OK)
|
||||
clone ruby "rails/rails" # 500K+ LOC, the Ruby framework
|
||||
clone lua "neovim/neovim" # 500K+ LOC, editor
|
||||
clone bash "ohmyzsh/ohmyzsh" # 100K+ .sh files
|
||||
clone zig "tigerbeetle/tigerbeetle" # 224K LOC (already large)
|
||||
clone haskell "jgm/pandoc" # 433K LOC (already large)
|
||||
clone ocaml "ocaml/dune" # 345K LOC (already large)
|
||||
clone elixir "plausible/analytics" # 677K LOC (already large)
|
||||
clone erlang "emqx/emqx" # 500K+ LOC, MQTT broker
|
||||
clone objc "realm/realm-cocoa" # 200K+ LOC, database SDK
|
||||
clone swift "Alamofire/Alamofire" # 370K LOC (already large)
|
||||
clone dart "felangel/bloc" # 285K LOC (already large)
|
||||
clone perl "movabletype/movabletype" # 300K+ LOC, CMS
|
||||
clone groovy "spockframework/spock" # 137K LOC (OK)
|
||||
clone r "tidyverse/ggplot2" # 150K+ LOC, visualization
|
||||
clone clojure "clojure/clojure" # 108K LOC (OK)
|
||||
clone fsharp "dotnet/fsharp" # 500K+ LOC, the F# compiler
|
||||
clone julia "JuliaLang/julia" # 1M+ LOC, the Julia runtime
|
||||
clone vimscript "SpaceVim/SpaceVim" # 2.6M LOC (already huge)
|
||||
clone nix "NixOS/nixpkgs" # 6M LOC (already huge)
|
||||
clone commonlisp "lem-project/lem" # 1.2M LOC (already large)
|
||||
clone elm "elm/compiler" # 57K LOC (largest Elm repo available)
|
||||
clone fortran "cp2k/cp2k" # 5.9M LOC (already huge)
|
||||
clone cobol "OCamlPro/gnucobol" # 540K LOC (already large)
|
||||
clone verilog "YosysHQ/yosys" # 517K LOC (already large)
|
||||
clone emacslisp "emacs-mirror/emacs" # 5.3M LOC (already huge)
|
||||
clone matlab "acristoffers/tree-sitter-matlab" # 133K LOC (best available)
|
||||
clone lean "leanprover-community/mathlib4" # 2.3M LOC (already huge)
|
||||
clone form "vermaseren/form" # 221K LOC (already large)
|
||||
clone wolfram "WolframResearch/WolframLanguageForJupyter" # 4K LOC (largest public Wolfram repo)
|
||||
|
||||
# Helper languages — Tier 2 (22 languages)
|
||||
clone yaml "kubernetes/examples" # K8s manifests
|
||||
clone hcl "hashicorp/terraform-provider-aws" # 1M+ LOC, massive HCL
|
||||
clone scss "twbs/bootstrap" # 120K LOC (OK)
|
||||
clone dockerfile "docker-library/official-images" # Docker configs
|
||||
clone cmake "Kitware/CMake" # 1.7M LOC (already huge)
|
||||
clone protobuf "googleapis/googleapis" # 2.2M LOC (already huge)
|
||||
clone graphql "graphql/graphql-spec" # 23K LOC (largest pure GraphQL)
|
||||
clone vue "vuejs/core" # 200K+ LOC, Vue 3 core
|
||||
clone svelte "sveltejs/svelte" # 267K LOC (already large)
|
||||
clone meson "mesonbuild/meson" # 237K LOC (already large)
|
||||
|
||||
# Shared repos (symlinked — language uses same repo as primary)
|
||||
symlink html javascript # Express views contain HTML
|
||||
symlink css tsx # shadcn-ui styles
|
||||
symlink toml rust # meilisearch Cargo.toml + config
|
||||
symlink sql java # spring-petclinic SQL schemas
|
||||
clone cuda "NVIDIA/cuda-samples"
|
||||
symlink json typescript # trpc JSON configs
|
||||
symlink xml java # spring-petclinic XML configs
|
||||
symlink markdown python # httpie docs
|
||||
symlink makefile c # redis Makefile
|
||||
clone glsl "repalash/Open-Shaders"
|
||||
symlink ini python # httpie .cfg/.ini files
|
||||
symlink magma lean # .m files — disambiguated via content markers
|
||||
symlink kubernetes yaml # YAML subtype — Deployment/Service manifests
|
||||
symlink kustomize yaml # YAML subtype — kustomization.yaml
|
||||
|
||||
echo ""
|
||||
echo "=== Clone complete ==="
|
||||
ls -1 "$BENCH_DIR/" | wc -l | xargs printf "%s repos ready in $BENCH_DIR\n"
|
||||
Executable
+194
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env bash
|
||||
# embed-frontend.sh — Convert built frontend assets into linkable object files.
|
||||
#
|
||||
# Usage: scripts/embed-frontend.sh <dist_dir> <output_dir>
|
||||
#
|
||||
# For each file in dist_dir, creates:
|
||||
# 1. An object file via `ld -r -b binary` (raw bytes, zero bloat)
|
||||
# 2. A generated embedded_assets.c with a lookup table
|
||||
#
|
||||
# Symbols created per file: _binary_<mangled_name>_start, _binary_<mangled_name>_end
|
||||
# On macOS, these get a leading underscore: __binary_<mangled_name>_start
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DIST_DIR="${1:?Usage: embed-frontend.sh <dist_dir> <output_dir>}"
|
||||
OUTPUT_DIR="${2:?Usage: embed-frontend.sh <dist_dir> <output_dir>}"
|
||||
|
||||
# Clean old embedded objects (asset hashes change on each build)
|
||||
rm -rf "$OUTPUT_DIR"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Detect platform — Linux uses ld -r -b binary, everything else uses xxd+cc
|
||||
IS_LINUX=false
|
||||
if [[ "$(uname -s)" == "Linux" ]] && ! [[ "$(uname -s)" =~ MINGW|MSYS ]]; then
|
||||
IS_LINUX=true
|
||||
fi
|
||||
|
||||
# Content-type detection
|
||||
content_type_for() {
|
||||
local f="$1"
|
||||
case "$f" in
|
||||
*.html) echo "text/html" ;;
|
||||
*.js) echo "application/javascript" ;;
|
||||
*.css) echo "text/css" ;;
|
||||
*.json) echo "application/json" ;;
|
||||
*.svg) echo "image/svg+xml" ;;
|
||||
*.png) echo "image/png" ;;
|
||||
*.ico) echo "image/x-icon" ;;
|
||||
*.woff2) echo "font/woff2" ;;
|
||||
*.woff) echo "font/woff" ;;
|
||||
*.map) echo "application/json" ;;
|
||||
*) echo "application/octet-stream" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Mangle filename to valid C symbol: replace non-alnum with _
|
||||
mangle() {
|
||||
echo "$1" | sed 's/[^a-zA-Z0-9]/_/g'
|
||||
}
|
||||
|
||||
# Collect all files
|
||||
FILES=()
|
||||
while IFS= read -r -d '' file; do
|
||||
FILES+=("$file")
|
||||
done < <(find "$DIST_DIR" -type f -print0 | sort -z)
|
||||
|
||||
if [[ ${#FILES[@]} -eq 0 ]]; then
|
||||
echo "Error: no files found in $DIST_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Embedding ${#FILES[@]} files from $DIST_DIR"
|
||||
|
||||
# Generate object files
|
||||
OBJ_FILES=()
|
||||
for file in "${FILES[@]}"; do
|
||||
rel="${file#$DIST_DIR/}"
|
||||
mangled=$(mangle "$rel")
|
||||
obj="$OUTPUT_DIR/embed_${mangled}.o"
|
||||
|
||||
if $IS_LINUX; then
|
||||
# Linux: ld -r -b binary (zero bloat, ELF only)
|
||||
abs_obj="$(cd "$(dirname "$0")/.." && pwd)/$obj"
|
||||
(cd "$DIST_DIR" && ld -r -b binary -o "$abs_obj" "$rel")
|
||||
else
|
||||
# macOS/Windows/MSYS2: generate C byte array + cc (no xxd dependency)
|
||||
local_c="$OUTPUT_DIR/embed_${mangled}.c"
|
||||
local_sym="_binary_${mangled}"
|
||||
|
||||
echo "/* Generated from $rel */" > "$local_c"
|
||||
echo "const unsigned char ${local_sym}_data[] = {" >> "$local_c"
|
||||
# Use od (POSIX) to generate hex bytes — works everywhere without xxd/vim
|
||||
od -An -tx1 -v < "$file" | tr -s ' ' '\n' | grep -v '^$' | sed 's/^/0x/; s/$/,/' | paste -sd' ' - | fold -s -w 76 | sed 's/^/ /' >> "$local_c"
|
||||
echo "};" >> "$local_c"
|
||||
echo "const unsigned int ${local_sym}_size = sizeof(${local_sym}_data);" >> "$local_c"
|
||||
|
||||
${CC:-cc} -c -O2 -o "$obj" "$local_c"
|
||||
rm -f "$local_c"
|
||||
fi
|
||||
|
||||
OBJ_FILES+=("$obj")
|
||||
echo " $rel -> $obj"
|
||||
done
|
||||
|
||||
# Generate embedded_assets.c
|
||||
ASSETS_C="src/ui/embedded_assets.c"
|
||||
cat > "$ASSETS_C" <<'HEADER'
|
||||
/*
|
||||
* embedded_assets.c — Generated file mapping URL paths to embedded bytes.
|
||||
* DO NOT EDIT — regenerated by scripts/embed-frontend.sh
|
||||
*/
|
||||
#include "ui/embedded_assets.h"
|
||||
#include <string.h>
|
||||
|
||||
HEADER
|
||||
|
||||
# Declare extern symbols
|
||||
for file in "${FILES[@]}"; do
|
||||
rel="${file#$DIST_DIR/}"
|
||||
mangled=$(mangle "$rel")
|
||||
sym="_binary_${mangled}"
|
||||
|
||||
if ! $IS_LINUX; then
|
||||
echo "extern const unsigned char ${sym}_data[];" >> "$ASSETS_C"
|
||||
echo "extern const unsigned int ${sym}_size;" >> "$ASSETS_C"
|
||||
else
|
||||
echo "extern const unsigned char ${sym}_start[];" >> "$ASSETS_C"
|
||||
echo "extern const unsigned char ${sym}_end[];" >> "$ASSETS_C"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "" >> "$ASSETS_C"
|
||||
echo "cbm_embedded_file_t CBM_EMBEDDED_FILES[] = {" >> "$ASSETS_C"
|
||||
|
||||
for file in "${FILES[@]}"; do
|
||||
rel="${file#$DIST_DIR/}"
|
||||
mangled=$(mangle "$rel")
|
||||
sym="_binary_${mangled}"
|
||||
ct=$(content_type_for "$rel")
|
||||
|
||||
# URL path: /index.html for root, /assets/... for assets
|
||||
url_path="/$rel"
|
||||
|
||||
if ! $IS_LINUX; then
|
||||
echo " {\"$url_path\", ${sym}_data, 0, \"$ct\"}," >> "$ASSETS_C"
|
||||
else
|
||||
echo " {\"$url_path\", ${sym}_start, 0, \"$ct\"}," >> "$ASSETS_C"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "};" >> "$ASSETS_C"
|
||||
echo "const int CBM_EMBEDDED_FILE_COUNT = ${#FILES[@]};" >> "$ASSETS_C"
|
||||
|
||||
# Generate size fixup
|
||||
if $IS_LINUX; then
|
||||
# Linux: compute size from start/end pointers (ld -r -b binary symbols)
|
||||
cat >> "$ASSETS_C" <<'SIZEINIT'
|
||||
|
||||
static void __attribute__((constructor)) init_embedded_sizes(void) {
|
||||
cbm_embedded_file_t *files = CBM_EMBEDDED_FILES;
|
||||
SIZEINIT
|
||||
|
||||
for i in "${!FILES[@]}"; do
|
||||
rel="${FILES[$i]#$DIST_DIR/}"
|
||||
mangled=$(mangle "$rel")
|
||||
sym="_binary_${mangled}"
|
||||
echo " files[$i].size = (unsigned int)(${sym}_end - ${sym}_start);" >> "$ASSETS_C"
|
||||
done
|
||||
|
||||
echo "}" >> "$ASSETS_C"
|
||||
else
|
||||
# macOS/Windows: use explicit _size vars from xxd-generated C arrays
|
||||
cat >> "$ASSETS_C" <<'SIZEINIT'
|
||||
|
||||
static void __attribute__((constructor)) init_embedded_sizes(void) {
|
||||
cbm_embedded_file_t *files = CBM_EMBEDDED_FILES;
|
||||
SIZEINIT
|
||||
|
||||
for i in "${!FILES[@]}"; do
|
||||
rel="${FILES[$i]#$DIST_DIR/}"
|
||||
mangled=$(mangle "$rel")
|
||||
sym="_binary_${mangled}"
|
||||
echo " files[$i].size = ${sym}_size;" >> "$ASSETS_C"
|
||||
done
|
||||
|
||||
echo "}" >> "$ASSETS_C"
|
||||
fi
|
||||
|
||||
# Add lookup function
|
||||
cat >> "$ASSETS_C" <<'LOOKUP'
|
||||
|
||||
const cbm_embedded_file_t *cbm_embedded_lookup(const char *path) {
|
||||
for (int i = 0; i < CBM_EMBEDDED_FILE_COUNT; i++) {
|
||||
if (strcmp(CBM_EMBEDDED_FILES[i].path, path) == 0) {
|
||||
return &CBM_EMBEDDED_FILES[i];
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
LOOKUP
|
||||
|
||||
echo "Generated $ASSETS_C with ${#FILES[@]} embedded files"
|
||||
echo "Object files in $OUTPUT_DIR:"
|
||||
printf ' %s\n' "${OBJ_FILES[@]}"
|
||||
Executable
+105
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
# env.sh — Shared environment detection for all build scripts.
|
||||
#
|
||||
# Sourced by test.sh, build.sh, lint.sh. Not meant to run standalone.
|
||||
#
|
||||
# Exports:
|
||||
# ARCH — target architecture (arm64 / x86_64)
|
||||
# ARCHFLAGS — "-arch <arch>" on macOS (target slice for clang/ld), empty elsewhere
|
||||
# NPROC — number of CPU cores
|
||||
# OS — darwin / linux / windows
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Detect OS ──────────────────────────────────────────────────
|
||||
OS="$(uname -s | tr '[:upper:]' '[:lower:]')"
|
||||
case "$OS" in
|
||||
darwin) OS="darwin" ;;
|
||||
linux) OS="linux" ;;
|
||||
mingw*|msys*|cygwin*) OS="windows" ;;
|
||||
*) OS="unknown" ;;
|
||||
esac
|
||||
|
||||
# ── Detect / override architecture ─────────────────────────────
|
||||
# Default: native HARDWARE architecture (not Rosetta-translated).
|
||||
# On macOS under Rosetta, uname -m returns x86_64 even on Apple Silicon.
|
||||
# We use sysctl to detect the true hardware.
|
||||
HW_ARCH="$(uname -m)"
|
||||
if [[ "$(uname -s)" == "Darwin" ]] && sysctl -n hw.optional.arm64 2>/dev/null | grep -q 1; then
|
||||
HW_ARCH="arm64"
|
||||
fi
|
||||
case "$HW_ARCH" in
|
||||
aarch64|arm64) HW_ARCH="arm64" ;;
|
||||
x86_64|amd64) HW_ARCH="x86_64" ;;
|
||||
esac
|
||||
|
||||
# CBM_ARCH env var or --arch flag override (parsed by calling script)
|
||||
ARCH="${CBM_ARCH:-$HW_ARCH}"
|
||||
|
||||
# ── Target-architecture flags (macOS only) ─────────────────────
|
||||
# Select the target slice explicitly with clang/ld's -arch instead of
|
||||
# relaunching make under `arch -<arch>`. Explicit -arch is the only approach
|
||||
# that works across toolchains: a Nix/Homebrew clang wrapper has a fixed
|
||||
# target triple, so `arch -x86_64 make` would still emit native arm64. It also
|
||||
# lets host tools (node, codegen) keep running natively during a cross-build.
|
||||
# Makefile.cbm folds $(ARCHFLAGS) into CC/CXX so it reaches every compile and
|
||||
# link, including the vendored objects. Empty on Linux/Windows.
|
||||
ARCHFLAGS=""
|
||||
if [[ "$OS" == "darwin" ]]; then
|
||||
ARCHFLAGS="-arch ${ARCH}"
|
||||
fi
|
||||
export ARCHFLAGS
|
||||
|
||||
# ── Detect parallelism ─────────────────────────────────────────
|
||||
NPROC=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
|
||||
|
||||
# ── Verify compiler can build for the target arch ──────────────
|
||||
# On macOS, PROBE actual capability instead of inspecting the compiler's own
|
||||
# Mach-O header. The driver is often a wrapper script (Nix, ccache, Homebrew)
|
||||
# or a clang that cross-compiles, so `file` on the binary says nothing about
|
||||
# what it can target — it just sees a shell script or a host-arch executable.
|
||||
# Compiling + linking a trivial program with -arch <target> is the truth.
|
||||
verify_compiler() {
|
||||
local compiler="$1"
|
||||
local bin
|
||||
bin="$(command -v "$compiler" 2>/dev/null || true)"
|
||||
|
||||
if [[ -z "$bin" ]]; then
|
||||
echo "ERROR: compiler '$compiler' not found in PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$OS" == "darwin" ]]; then
|
||||
local probe_out
|
||||
probe_out="$(mktemp -t cbm-archprobe.XXXXXX)"
|
||||
if ! printf 'int main(void){return 0;}\n' \
|
||||
| "$compiler" -arch "$ARCH" -x c - -o "$probe_out" >/dev/null 2>&1; then
|
||||
rm -f "$probe_out"
|
||||
echo "ERROR: $compiler cannot build for -arch $ARCH ($bin)" >&2
|
||||
echo " A trivial $ARCH compile + link failed with this toolchain." >&2
|
||||
if [[ "$ARCH" != "$HW_ARCH" ]]; then
|
||||
echo " Cross-building $HW_ARCH -> $ARCH needs the $ARCH SDK slice + runtime;" >&2
|
||||
echo " to build for this machine instead, re-run with: --arch $HW_ARCH" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$probe_out"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Default compiler selection ─────────────────────────────────
|
||||
# macOS: cc (Apple Clang). Linux/Windows: gcc (system default).
|
||||
# CI overrides via CC=gcc CXX=g++ args. Local macOS overrides via CC=cc.
|
||||
if [[ -z "${CC:-}" ]]; then
|
||||
if [[ "$OS" == "darwin" ]]; then
|
||||
export CC=cc CXX=c++
|
||||
else
|
||||
export CC=gcc CXX=g++
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Print environment summary ──────────────────────────────────
|
||||
print_env() {
|
||||
local context="$1"
|
||||
echo "=== $context: os=$OS arch=$ARCH cores=$NPROC cc=${CC:-default} cxx=${CXX:-default} ==="
|
||||
}
|
||||
Executable
+528
@@ -0,0 +1,528 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Extract token embeddings from nomic-embed-code (7B) for static lookup table.
|
||||
|
||||
Loads the full model, filters the vocabulary to code-relevant tokens,
|
||||
runs full inference on each token, applies simulated attention, quantizes
|
||||
to int8, and outputs files compatible with vendored/unixcoder/ format.
|
||||
|
||||
Usage:
|
||||
pip3.9 install torch transformers sentence-transformers
|
||||
python3.9 scripts/extract_nomic_vectors.py [--output-dir vendored/nomic]
|
||||
|
||||
Output:
|
||||
code_vectors.bin — [int32 count][int32 dim] + count×dim int8
|
||||
code_tokens.txt — one token per line
|
||||
code_tokens.h — C header: static const char *PRETRAINED_TOKENS[N]
|
||||
code_vectors.h — C header: defines + inline accessor
|
||||
code_vectors_blob.S — assembler .incbin
|
||||
|
||||
One-time extraction. ~2-3h on GPU, ~6-10h on M3 Pro CPU (float16, ~14GB RAM).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
# Parallelize CPU inference across all cores BEFORE any torch ops
|
||||
NUM_THREADS = min(os.cpu_count() * 2, 12)
|
||||
torch.set_num_threads(NUM_THREADS)
|
||||
torch.set_num_interop_threads(max(NUM_THREADS // 2, 1))
|
||||
os.environ.setdefault("OMP_NUM_THREADS", str(NUM_THREADS))
|
||||
os.environ.setdefault("MKL_NUM_THREADS", str(NUM_THREADS))
|
||||
|
||||
from transformers import AutoModel, AutoTokenizer
|
||||
|
||||
|
||||
# ── Configuration ──────────────────────────────────────────────────────
|
||||
|
||||
MODEL_NAME = "nomic-ai/nomic-embed-code"
|
||||
OUTPUT_DIM = 768 # Target dimension (Matryoshka truncation if model outputs more)
|
||||
SIM_ATTENTION_K = 32 # Top-K neighbors for simulated attention
|
||||
SIM_ATTENTION_ITERS = 3 # Number of simulated attention iterations
|
||||
SIM_ATTENTION_ALPHA = 0.3 # Blend ratio: (1-α)×original + α×neighbor_mean
|
||||
BATCH_SIZE = 32 # Tokens per inference batch (sized for thread saturation)
|
||||
CHECKPOINT_EVERY = 500 # Save checkpoint every N tokens
|
||||
|
||||
|
||||
# ── Token filtering ───────────────────────────────────────────────────
|
||||
|
||||
def is_code_relevant(token_str: str) -> bool:
|
||||
"""Filter vocabulary to code-relevant tokens.
|
||||
|
||||
Goal: keep tokens that our runtime camelCase/snake_case splitter would
|
||||
produce from identifiers. Reject BPE noise, punctuation combos, and
|
||||
non-Latin scripts.
|
||||
"""
|
||||
s = token_str.strip()
|
||||
if not s:
|
||||
return False
|
||||
|
||||
# Remove BPE markers (Ġ = space prefix, ▁ = sentencepiece, Ċ/ċ = newline in Qwen)
|
||||
clean = s.lstrip("\u0120\u2581") # Ġ, ▁
|
||||
if not clean:
|
||||
return False
|
||||
|
||||
# Skip special tokens
|
||||
if clean.startswith("<") and clean.endswith(">"):
|
||||
return False
|
||||
if clean.startswith("[") and clean.endswith("]"):
|
||||
return False
|
||||
|
||||
# Strip leading/trailing underscores (common in BPE) but keep content
|
||||
inner = clean.strip("_")
|
||||
if not inner:
|
||||
return False
|
||||
|
||||
# STRICT: must be purely alphanumeric + underscores (identifier-shaped)
|
||||
# This rejects BPE noise like "!");ċ", "!!!!ċċ", etc.
|
||||
if not re.match(r'^[a-zA-Z][a-zA-Z0-9_]*$', inner):
|
||||
return False
|
||||
|
||||
# Must be at least 2 chars of actual content
|
||||
if len(inner) < 2:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def clean_token(token_str: str) -> str:
|
||||
"""Normalize a BPE token to the form our runtime tokenizer produces."""
|
||||
s = token_str.strip()
|
||||
# Strip BPE space markers
|
||||
s = s.lstrip("\u0120\u2581")
|
||||
# Strip leading/trailing underscores
|
||||
s = s.strip("_")
|
||||
# Lowercase (our runtime tokenizer lowercases)
|
||||
s = s.lower()
|
||||
return s
|
||||
|
||||
|
||||
# ── Simulated attention ──────────────────────────────────────────────
|
||||
|
||||
def simulated_attention(vectors: np.ndarray, k: int, iterations: int,
|
||||
alpha: float) -> np.ndarray:
|
||||
"""
|
||||
Apply simulated self-attention: for each vector, blend with mean of
|
||||
top-K nearest neighbors. This approximates contextual composition
|
||||
that real attention provides.
|
||||
|
||||
vectors: (N, D) float32 unit-normalized
|
||||
Returns: (N, D) float32 unit-normalized
|
||||
"""
|
||||
n, d = vectors.shape
|
||||
result = vectors.copy()
|
||||
|
||||
for iteration in range(iterations):
|
||||
t0 = time.time()
|
||||
# Compute cosine similarity matrix in chunks to avoid OOM
|
||||
# For 40K vectors × 768d, full matrix = 40K² × 4 bytes = 6.4GB
|
||||
# Process in chunks of 2048
|
||||
chunk_size = 2048
|
||||
new_result = np.zeros_like(result)
|
||||
|
||||
for i in range(0, n, chunk_size):
|
||||
end = min(i + chunk_size, n)
|
||||
chunk = result[i:end] # (chunk, D)
|
||||
|
||||
# Cosine similarity: chunk × all^T
|
||||
sims = chunk @ result.T # (chunk, N)
|
||||
|
||||
# For each vector in chunk, find top-K neighbors (excluding self)
|
||||
for j in range(end - i):
|
||||
global_idx = i + j
|
||||
sim_row = sims[j].copy()
|
||||
sim_row[global_idx] = -1.0 # Exclude self
|
||||
|
||||
# Top-K indices
|
||||
if k < n - 1:
|
||||
top_k_idx = np.argpartition(sim_row, -k)[-k:]
|
||||
else:
|
||||
top_k_idx = np.arange(n)
|
||||
top_k_idx = top_k_idx[top_k_idx != global_idx]
|
||||
|
||||
neighbor_mean = result[top_k_idx].mean(axis=0)
|
||||
|
||||
# Blend
|
||||
blended = (1 - alpha) * result[global_idx] + alpha * neighbor_mean
|
||||
# Re-normalize
|
||||
norm = np.linalg.norm(blended)
|
||||
if norm > 1e-8:
|
||||
blended /= norm
|
||||
new_result[global_idx] = blended
|
||||
|
||||
result = new_result
|
||||
elapsed = time.time() - t0
|
||||
print(f" sim-attention iter {iteration + 1}/{iterations}: {elapsed:.1f}s")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ── Extraction ───────────────────────────────────────────────────────
|
||||
|
||||
def extract_embeddings(model, tokenizer, tokens: list, device: str,
|
||||
batch_size: int = 64,
|
||||
checkpoint_path: str = None) -> np.ndarray:
|
||||
"""Run full model inference on each token string. Returns (N, D) float32."""
|
||||
|
||||
# Check for checkpoint
|
||||
start_idx = 0
|
||||
all_vecs = []
|
||||
if checkpoint_path and os.path.exists(checkpoint_path):
|
||||
data = np.load(checkpoint_path)
|
||||
all_vecs = list(data["vectors"])
|
||||
start_idx = len(all_vecs)
|
||||
print(f" resuming from checkpoint: {start_idx}/{len(tokens)} tokens")
|
||||
|
||||
model.eval()
|
||||
total = len(tokens)
|
||||
t0 = time.time()
|
||||
|
||||
with torch.no_grad():
|
||||
for batch_start in range(start_idx, total, batch_size):
|
||||
batch_end = min(batch_start + batch_size, total)
|
||||
batch_tokens = tokens[batch_start:batch_end]
|
||||
|
||||
# nomic-embed-code requires search_query or search_document prefix
|
||||
# For single tokens, we use the token as-is (query mode)
|
||||
texts = [f"search_query: {t}" for t in batch_tokens]
|
||||
|
||||
encoded = tokenizer(
|
||||
texts,
|
||||
padding=True,
|
||||
truncation=True,
|
||||
max_length=64,
|
||||
return_tensors="pt"
|
||||
).to(device)
|
||||
|
||||
outputs = model(**encoded)
|
||||
|
||||
# Mean pooling over non-padding tokens
|
||||
attention_mask = encoded["attention_mask"]
|
||||
token_embeddings = outputs.last_hidden_state
|
||||
input_mask_expanded = (
|
||||
attention_mask.unsqueeze(-1)
|
||||
.expand(token_embeddings.size())
|
||||
.float()
|
||||
)
|
||||
sum_embeddings = torch.sum(
|
||||
token_embeddings * input_mask_expanded, dim=1
|
||||
)
|
||||
sum_mask = torch.clamp(input_mask_expanded.sum(dim=1), min=1e-9)
|
||||
mean_pooled = sum_embeddings / sum_mask
|
||||
|
||||
# Truncate to OUTPUT_DIM if model outputs more (Matryoshka)
|
||||
if mean_pooled.shape[1] > OUTPUT_DIM:
|
||||
mean_pooled = mean_pooled[:, :OUTPUT_DIM]
|
||||
|
||||
# L2 normalize
|
||||
mean_pooled = torch.nn.functional.normalize(mean_pooled, p=2, dim=1)
|
||||
|
||||
vecs = mean_pooled.cpu().numpy()
|
||||
all_vecs.extend(vecs)
|
||||
|
||||
# Progress
|
||||
done = batch_end
|
||||
elapsed = time.time() - t0
|
||||
rate = (done - start_idx) / elapsed if elapsed > 0 else 0
|
||||
eta = (total - done) / rate if rate > 0 else 0
|
||||
print(
|
||||
f" [{done:>6}/{total}] "
|
||||
f"{rate:.1f} tok/s "
|
||||
f"ETA {eta / 60:.0f}m",
|
||||
flush=True
|
||||
)
|
||||
|
||||
# Checkpoint
|
||||
if checkpoint_path and (done % CHECKPOINT_EVERY < batch_size):
|
||||
np.savez_compressed(
|
||||
checkpoint_path,
|
||||
vectors=np.array(all_vecs, dtype=np.float32)
|
||||
)
|
||||
|
||||
print()
|
||||
return np.array(all_vecs, dtype=np.float32)
|
||||
|
||||
|
||||
# ── Output generation ────────────────────────────────────────────────
|
||||
|
||||
def write_bin(path: str, vectors: np.ndarray, dim: int):
|
||||
"""Write binary blob: [int32 count][int32 dim] + count×dim int8."""
|
||||
n = vectors.shape[0]
|
||||
# Quantize: scale to [-127, 127], round to int8
|
||||
quantized = np.clip(np.round(vectors * 127.0), -127, 127).astype(np.int8)
|
||||
|
||||
with open(path, "wb") as f:
|
||||
f.write(struct.pack("<ii", n, dim))
|
||||
f.write(quantized.tobytes())
|
||||
|
||||
size_mb = os.path.getsize(path) / (1024 * 1024)
|
||||
print(f" {path}: {n} vectors × {dim}d = {size_mb:.1f} MB")
|
||||
|
||||
|
||||
def write_tokens_txt(path: str, tokens: list):
|
||||
"""Write plain text token list."""
|
||||
with open(path, "w") as f:
|
||||
for t in tokens:
|
||||
f.write(t + "\n")
|
||||
print(f" {path}: {len(tokens)} tokens")
|
||||
|
||||
|
||||
def write_tokens_h(path: str, tokens: list):
|
||||
"""Write C header with token string array."""
|
||||
with open(path, "w") as f:
|
||||
f.write(f"/* nomic-embed-code token vocabulary — {len(tokens)} tokens. */\n")
|
||||
f.write("#ifndef CBM_NOMIC_TOKENS_H\n")
|
||||
f.write("#define CBM_NOMIC_TOKENS_H\n\n")
|
||||
f.write(f"static const char *PRETRAINED_TOKENS[{len(tokens)}] = {{\n")
|
||||
for t in tokens:
|
||||
escaped = t.replace("\\", "\\\\").replace('"', '\\"')
|
||||
f.write(f'"{escaped}",\n')
|
||||
f.write("};\n\n")
|
||||
f.write("#endif /* CBM_NOMIC_TOKENS_H */\n")
|
||||
print(f" {path}: written")
|
||||
|
||||
|
||||
def write_vectors_h(path: str, token_count: int, dim: int, incbin_path: str):
|
||||
"""Write C header with defines and inline accessor."""
|
||||
with open(path, "w") as f:
|
||||
f.write(f"""/* nomic-embed-code (nomic-ai/nomic-embed-code) token embeddings.
|
||||
* {token_count} tokens x {dim}d int8-quantized unit vectors.
|
||||
* Distilled from 7B model via full inference on filtered vocabulary.
|
||||
* Simulated attention: {SIM_ATTENTION_ITERS} iterations, K={SIM_ATTENTION_K}, alpha={SIM_ATTENTION_ALPHA}.
|
||||
*
|
||||
* Vector blob embedded via code_vectors_blob.S (assembler .incbin).
|
||||
* Token strings are in this header as a static array.
|
||||
*
|
||||
* Source: https://huggingface.co/nomic-ai/nomic-embed-code
|
||||
* License: Apache 2.0
|
||||
*/
|
||||
#ifndef CBM_NOMIC_VECTORS_H
|
||||
#define CBM_NOMIC_VECTORS_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define PRETRAINED_TOKEN_COUNT {token_count}
|
||||
#define PRETRAINED_DIM {dim}
|
||||
|
||||
/* Raw vector blob: first 8 bytes = [int32 count][int32 dim],
|
||||
* then count x dim int8 values (unit-normalized, x127 scaled). */
|
||||
extern const unsigned char PRETRAINED_VECTOR_BLOB[];
|
||||
extern const unsigned int PRETRAINED_VECTOR_BLOB_LEN;
|
||||
|
||||
/* Access the int8 vector for token index i. */
|
||||
static inline const int8_t *pretrained_vec_at(int i) {{
|
||||
return (const int8_t *)(PRETRAINED_VECTOR_BLOB + 8 + (size_t)i * PRETRAINED_DIM);
|
||||
}}
|
||||
|
||||
/* Token strings (separate header to keep this file clean). */
|
||||
#include "code_tokens.h"
|
||||
|
||||
#endif /* CBM_NOMIC_VECTORS_H */
|
||||
""")
|
||||
print(f" {path}: written")
|
||||
|
||||
|
||||
def write_blob_s(path: str, incbin_path: str):
|
||||
"""Write assembler .incbin directive."""
|
||||
with open(path, "w") as f:
|
||||
f.write(f"""/* nomic-embed-code vector blob embedded via assembler. */
|
||||
.section __DATA,__const
|
||||
.globl _PRETRAINED_VECTOR_BLOB
|
||||
.globl _PRETRAINED_VECTOR_BLOB_LEN
|
||||
.p2align 4
|
||||
_PRETRAINED_VECTOR_BLOB:
|
||||
.incbin "{incbin_path}"
|
||||
_PRETRAINED_VECTOR_BLOB_END:
|
||||
|
||||
.section __DATA,__const
|
||||
.p2align 2
|
||||
_PRETRAINED_VECTOR_BLOB_LEN:
|
||||
.long _PRETRAINED_VECTOR_BLOB_END - _PRETRAINED_VECTOR_BLOB
|
||||
""")
|
||||
print(f" {path}: written")
|
||||
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Extract nomic-embed-code token embeddings")
|
||||
parser.add_argument("--output-dir", default="vendored/nomic",
|
||||
help="Output directory (default: vendored/nomic)")
|
||||
parser.add_argument("--device", default=None,
|
||||
help="Device: cuda, mps, cpu (auto-detected)")
|
||||
parser.add_argument("--skip-attention", action="store_true",
|
||||
help="Skip simulated attention (faster, lower quality)")
|
||||
parser.add_argument("--batch-size", type=int, default=BATCH_SIZE,
|
||||
help=f"Batch size (default: {BATCH_SIZE})")
|
||||
parser.add_argument("--checkpoint", default=None,
|
||||
help="Checkpoint file path (auto: <output-dir>/checkpoint.npz)")
|
||||
args = parser.parse_args()
|
||||
|
||||
batch_size = args.batch_size
|
||||
|
||||
# Auto-detect device
|
||||
# Prefer CPU for 7B models on Apple Silicon — MPS shares unified memory
|
||||
# with the system and can cause OOM/crashes. CPU keeps allocation predictable.
|
||||
# Use --device mps to override if you have enough headroom (32GB+).
|
||||
if args.device:
|
||||
device = args.device
|
||||
elif torch.cuda.is_available():
|
||||
device = "cuda"
|
||||
else:
|
||||
device = "cpu"
|
||||
|
||||
# Force line-buffered stdout so tee/log sees output immediately
|
||||
sys.stdout.reconfigure(line_buffering=True)
|
||||
|
||||
print(f"device={device}")
|
||||
print(f"threads={torch.get_num_threads()}")
|
||||
print(f"model={MODEL_NAME}")
|
||||
print(f"output_dim={OUTPUT_DIM}")
|
||||
print()
|
||||
|
||||
# Create output dir
|
||||
out_dir = Path(args.output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
checkpoint_path = args.checkpoint or str(out_dir / "checkpoint.npz")
|
||||
|
||||
# ── Step 1: Load model + tokenizer ──
|
||||
print("step 1: loading model + tokenizer...")
|
||||
t0 = time.time()
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
|
||||
model = AutoModel.from_pretrained(
|
||||
MODEL_NAME,
|
||||
trust_remote_code=True,
|
||||
dtype=torch.float16, # 7B×2B = ~14GB (vs 28GB float32)
|
||||
low_cpu_mem_usage=True, # Stream weights, no 2x peak during load
|
||||
)
|
||||
model = model.to(device)
|
||||
print(f" loaded in {time.time() - t0:.1f}s")
|
||||
print(f" hidden_size={model.config.hidden_size}")
|
||||
print(f" vocab_size={tokenizer.vocab_size}")
|
||||
print()
|
||||
|
||||
# ── Step 2: Filter vocabulary ──
|
||||
print("step 2: filtering vocabulary to code-relevant tokens...")
|
||||
vocab = tokenizer.get_vocab()
|
||||
print(f" raw vocabulary: {len(vocab)} tokens")
|
||||
|
||||
# Filter and deduplicate
|
||||
seen = set()
|
||||
filtered_tokens = []
|
||||
for tok_str, tok_id in sorted(vocab.items(), key=lambda x: x[1]):
|
||||
if not is_code_relevant(tok_str):
|
||||
continue
|
||||
clean = clean_token(tok_str)
|
||||
if not clean or clean in seen:
|
||||
continue
|
||||
if len(clean) < 2:
|
||||
continue
|
||||
seen.add(clean)
|
||||
filtered_tokens.append(clean)
|
||||
|
||||
filtered_tokens.sort()
|
||||
print(f" code-relevant (deduplicated): {len(filtered_tokens)} tokens")
|
||||
|
||||
# Show sample
|
||||
sample = filtered_tokens[:20]
|
||||
print(f" sample: {sample}")
|
||||
print()
|
||||
|
||||
# ── Step 3: Extract embeddings (full inference) ──
|
||||
print(f"step 3: extracting embeddings ({len(filtered_tokens)} tokens, batch_size={batch_size})...")
|
||||
t0 = time.time()
|
||||
vectors = extract_embeddings(
|
||||
model, tokenizer, filtered_tokens, device,
|
||||
batch_size=batch_size, checkpoint_path=checkpoint_path
|
||||
)
|
||||
elapsed = time.time() - t0
|
||||
print(f" extracted {vectors.shape[0]} vectors × {vectors.shape[1]}d in {elapsed:.0f}s")
|
||||
|
||||
# Truncate to OUTPUT_DIM if needed
|
||||
if vectors.shape[1] > OUTPUT_DIM:
|
||||
print(f" truncating {vectors.shape[1]}d -> {OUTPUT_DIM}d (Matryoshka)")
|
||||
vectors = vectors[:, :OUTPUT_DIM]
|
||||
# Re-normalize after truncation
|
||||
norms = np.linalg.norm(vectors, axis=1, keepdims=True)
|
||||
norms = np.maximum(norms, 1e-8)
|
||||
vectors = vectors / norms
|
||||
|
||||
print(f" final shape: {vectors.shape}")
|
||||
|
||||
# Mean-center to fix anisotropy (transformer embeddings cluster tightly,
|
||||
# making all cosine similarities ~0.95+). Subtracting the corpus mean
|
||||
# spreads vectors apart, making cosine discriminative.
|
||||
mean_vec = vectors.mean(axis=0)
|
||||
mean_norm = np.linalg.norm(mean_vec)
|
||||
print(f" mean vector norm before centering: {mean_norm:.4f} (>0.5 = anisotropic)")
|
||||
vectors = vectors - mean_vec
|
||||
# Re-normalize after centering
|
||||
norms = np.linalg.norm(vectors, axis=1, keepdims=True)
|
||||
norms = np.maximum(norms, 1e-8)
|
||||
vectors = vectors / norms
|
||||
mean_after = np.linalg.norm(vectors.mean(axis=0))
|
||||
print(f" mean vector norm after centering: {mean_after:.6f}")
|
||||
print()
|
||||
|
||||
# ── Step 4: Simulated attention ──
|
||||
if not args.skip_attention:
|
||||
print(f"step 4: simulated attention (K={SIM_ATTENTION_K}, "
|
||||
f"iters={SIM_ATTENTION_ITERS}, alpha={SIM_ATTENTION_ALPHA})...")
|
||||
t0 = time.time()
|
||||
vectors = simulated_attention(
|
||||
vectors, SIM_ATTENTION_K, SIM_ATTENTION_ITERS, SIM_ATTENTION_ALPHA
|
||||
)
|
||||
print(f" completed in {time.time() - t0:.1f}s")
|
||||
print()
|
||||
else:
|
||||
print("step 4: simulated attention SKIPPED")
|
||||
print()
|
||||
|
||||
# ── Step 5: Write output files ──
|
||||
print("step 5: writing output files...")
|
||||
dim = vectors.shape[1]
|
||||
|
||||
write_bin(str(out_dir / "code_vectors.bin"), vectors, dim)
|
||||
write_tokens_txt(str(out_dir / "code_tokens.txt"), filtered_tokens)
|
||||
write_tokens_h(str(out_dir / "code_tokens.h"), filtered_tokens)
|
||||
|
||||
incbin_path = f"vendored/nomic/code_vectors.bin"
|
||||
write_vectors_h(str(out_dir / "code_vectors.h"), len(filtered_tokens), dim, incbin_path)
|
||||
write_blob_s(str(out_dir / "code_vectors_blob.S"), incbin_path)
|
||||
print()
|
||||
|
||||
# Cleanup checkpoint
|
||||
if os.path.exists(checkpoint_path):
|
||||
os.remove(checkpoint_path)
|
||||
print(f" removed checkpoint: {checkpoint_path}")
|
||||
|
||||
# ── Summary ──
|
||||
bin_size = os.path.getsize(str(out_dir / "code_vectors.bin"))
|
||||
print()
|
||||
print("=" * 60)
|
||||
print(f" model: {MODEL_NAME}")
|
||||
print(f" tokens: {len(filtered_tokens)}")
|
||||
print(f" dimensions: {dim}")
|
||||
print(f" blob size: {bin_size / (1024*1024):.1f} MB")
|
||||
print(f" sim-attn: {'yes' if not args.skip_attention else 'no'}")
|
||||
print(f" output: {out_dir}/")
|
||||
print("=" * 60)
|
||||
print()
|
||||
print("next steps:")
|
||||
print(f" 1. update Makefile.cbm: change UNIXCODER_BLOB_SRC path to vendored/nomic/")
|
||||
print(f" 2. update #include in semantic.c: \"vendored/nomic/code_vectors.h\"")
|
||||
print(f" 3. arch -arm64 make -j12 -f Makefile.cbm clean-c && arch -arm64 make -j12 -f Makefile.cbm")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+432
@@ -0,0 +1,432 @@
|
||||
#!/usr/bin/env python3.9
|
||||
"""
|
||||
gen-py-stdlib.py — Generate cbm_python_stdlib_register from typeshed stubs.
|
||||
|
||||
Phase 10 of Python LSP integration. Walks a typeshed/stdlib checkout,
|
||||
parses each .pyi via the standard `ast` module, and emits a single C
|
||||
source file (internal/cbm/lsp/generated/python_stdlib_data.c) that
|
||||
populates a CBMTypeRegistry with classes + their method names and
|
||||
free functions per module.
|
||||
|
||||
This is a v1 implementation — overload stacks collapse to the first
|
||||
signature, ParamSpec/TypeVarTuple/Concatenate are skipped, version
|
||||
guards `if sys.version_info >= (X, Y):` are flattened (we take the
|
||||
union of all branches). Per-symbol min/max version guards from the
|
||||
plan are recorded as comments only in v1; py_lsp consumers should
|
||||
treat the generator output as the 3.12 baseline.
|
||||
|
||||
Usage:
|
||||
python3.9 scripts/gen-py-stdlib.py <typeshed-stdlib-path> <output.c>
|
||||
|
||||
Defaults to /tmp/python-lsp-references/typeshed/stdlib and
|
||||
internal/cbm/lsp/generated/python_stdlib_data.c when run without args.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import dataclasses
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
# v1 module allowlist — the plan's "top usage" set. Skip large / low-value
|
||||
# modules (tkinter, turtle, curses, xml.*, email.*) to keep generated
|
||||
# output manageable.
|
||||
ALLOWED_MODULES = {
|
||||
"builtins",
|
||||
"typing",
|
||||
"typing_extensions",
|
||||
"os",
|
||||
"sys",
|
||||
"collections",
|
||||
"functools",
|
||||
"itertools",
|
||||
"pathlib",
|
||||
"json",
|
||||
"re",
|
||||
"dataclasses",
|
||||
"enum",
|
||||
"datetime",
|
||||
"subprocess",
|
||||
"logging",
|
||||
"asyncio",
|
||||
"unittest",
|
||||
"argparse",
|
||||
"contextlib",
|
||||
"io",
|
||||
"tempfile",
|
||||
"shutil",
|
||||
"inspect",
|
||||
"abc",
|
||||
"warnings",
|
||||
"weakref",
|
||||
"copy",
|
||||
"pickle",
|
||||
"time",
|
||||
"math",
|
||||
"string",
|
||||
"socket",
|
||||
"threading",
|
||||
"multiprocessing",
|
||||
"queue",
|
||||
"urllib",
|
||||
"http",
|
||||
"concurrent",
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class StubMethod:
|
||||
name: str
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class StubClass:
|
||||
qualified_name: str
|
||||
short_name: str
|
||||
methods: list[str]
|
||||
bases: list[str]
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class StubFunction:
|
||||
qualified_name: str
|
||||
short_name: str
|
||||
module_qn: str
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ModuleStubs:
|
||||
module_qn: str
|
||||
classes: list[StubClass]
|
||||
functions: list[StubFunction]
|
||||
|
||||
|
||||
def is_method(node: ast.AST) -> bool:
|
||||
return isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
|
||||
|
||||
def class_methods(class_node: ast.ClassDef) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for child in class_node.body:
|
||||
if is_method(child):
|
||||
name = child.name # type: ignore[attr-defined]
|
||||
if not name.startswith("_") or name in {"__init__", "__call__", "__enter__", "__exit__"}:
|
||||
if name not in seen:
|
||||
seen.add(name)
|
||||
out.append(name)
|
||||
elif isinstance(child, ast.If):
|
||||
# Flatten version guards: walk both branches.
|
||||
for sub in child.body + child.orelse:
|
||||
if is_method(sub):
|
||||
name = sub.name # type: ignore[attr-defined]
|
||||
if name not in seen:
|
||||
seen.add(name)
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
def base_qns(class_node: ast.ClassDef, module_qn: str) -> list[str]:
|
||||
out: list[str] = []
|
||||
for base in class_node.bases:
|
||||
if isinstance(base, ast.Name):
|
||||
# Bare name — qualify within the same module by default.
|
||||
out.append(f"{module_qn}.{base.id}")
|
||||
elif isinstance(base, ast.Attribute):
|
||||
# mod.SubClass
|
||||
try:
|
||||
text = ast.unparse(base) # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
continue
|
||||
out.append(text)
|
||||
return out
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class StarReExport:
|
||||
target_module: str # e.g. "posixpath" / "ntpath"
|
||||
|
||||
|
||||
def parse_module(path: Path, module_qn: str) -> tuple[ModuleStubs, list[StarReExport]]:
|
||||
src = path.read_text(encoding="utf-8", errors="replace")
|
||||
try:
|
||||
tree = ast.parse(src)
|
||||
except SyntaxError:
|
||||
return ModuleStubs(module_qn=module_qn, classes=[], functions=[]), []
|
||||
|
||||
classes: list[StubClass] = []
|
||||
functions: list[StubFunction] = []
|
||||
star_imports: list[StarReExport] = []
|
||||
seen_funcs: set[str] = set()
|
||||
|
||||
def walk(body: list[ast.stmt], current_module_qn: str) -> None:
|
||||
for node in body:
|
||||
if isinstance(node, ast.ClassDef):
|
||||
qn = f"{current_module_qn}.{node.name}"
|
||||
classes.append(StubClass(
|
||||
qualified_name=qn,
|
||||
short_name=node.name,
|
||||
methods=class_methods(node),
|
||||
bases=base_qns(node, current_module_qn),
|
||||
))
|
||||
elif is_method(node):
|
||||
name = node.name # type: ignore[attr-defined]
|
||||
if name.startswith("_") and name not in {"__init__"}:
|
||||
continue
|
||||
if name in seen_funcs:
|
||||
continue
|
||||
seen_funcs.add(name)
|
||||
functions.append(StubFunction(
|
||||
qualified_name=f"{current_module_qn}.{name}",
|
||||
short_name=name,
|
||||
module_qn=current_module_qn,
|
||||
))
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
# Track `from X import *` for re-export resolution.
|
||||
if node.module and any(alias.name == "*" for alias in node.names):
|
||||
star_imports.append(StarReExport(target_module=node.module))
|
||||
elif isinstance(node, ast.If):
|
||||
# Flatten version guards.
|
||||
walk(node.body, current_module_qn)
|
||||
walk(node.orelse, current_module_qn)
|
||||
|
||||
walk(tree.body, module_qn)
|
||||
return ModuleStubs(module_qn=module_qn, classes=classes, functions=functions), star_imports
|
||||
|
||||
|
||||
def collect_all_stubs(stdlib_root: Path) -> tuple[dict[str, ModuleStubs], dict[str, list[StarReExport]]]:
|
||||
"""Gather every stub in the allowlist plus any re-export targets they
|
||||
pull in transitively (e.g. os.path -> posixpath -> genericpath).
|
||||
Returns (modules, star_imports_per_module). """
|
||||
modules: dict[str, ModuleStubs] = {}
|
||||
star_imports: dict[str, list[StarReExport]] = {}
|
||||
|
||||
# First pass: walk allowlist
|
||||
queue: list[tuple[Path, str]] = []
|
||||
for path in sorted(stdlib_root.rglob("*.pyi")):
|
||||
rel = path.relative_to(stdlib_root)
|
||||
parts = list(rel.parts)
|
||||
if parts[-1] == "__init__.pyi":
|
||||
parts.pop()
|
||||
else:
|
||||
parts[-1] = parts[-1][:-len(".pyi")]
|
||||
if not parts:
|
||||
continue
|
||||
top = parts[0]
|
||||
if top.startswith("_"):
|
||||
continue
|
||||
if top not in ALLOWED_MODULES:
|
||||
continue
|
||||
module_qn = ".".join(parts)
|
||||
queue.append((path, module_qn))
|
||||
|
||||
seen_targets: set[str] = set()
|
||||
while queue:
|
||||
path, mod_qn = queue.pop()
|
||||
if mod_qn in modules:
|
||||
continue
|
||||
ms, stars = parse_module(path, mod_qn)
|
||||
modules[mod_qn] = ms
|
||||
star_imports[mod_qn] = stars
|
||||
# Enqueue re-export targets (e.g. posixpath, ntpath, genericpath)
|
||||
for star in stars:
|
||||
tgt = star.target_module
|
||||
if tgt in seen_targets:
|
||||
continue
|
||||
seen_targets.add(tgt)
|
||||
tgt_path = stdlib_root / (tgt.replace(".", "/") + ".pyi")
|
||||
if tgt_path.is_file():
|
||||
queue.append((tgt_path, tgt))
|
||||
else:
|
||||
tgt_init = stdlib_root / tgt.replace(".", "/") / "__init__.pyi"
|
||||
if tgt_init.is_file():
|
||||
queue.append((tgt_init, tgt))
|
||||
|
||||
return modules, star_imports
|
||||
|
||||
|
||||
def resolve_reexports(modules: dict[str, ModuleStubs],
|
||||
star_imports: dict[str, list[StarReExport]]) -> None:
|
||||
"""For each module with `from X import *`, copy X's classes/functions
|
||||
into the current module under that module's QN. Iterates to a fixed
|
||||
point to handle re-export chains. """
|
||||
changed = True
|
||||
iter_count = 0
|
||||
while changed and iter_count < 8:
|
||||
changed = False
|
||||
iter_count += 1
|
||||
for mod_qn, stars in star_imports.items():
|
||||
ms = modules[mod_qn]
|
||||
for star in stars:
|
||||
target = modules.get(star.target_module)
|
||||
if not target:
|
||||
continue
|
||||
# Copy target's classes/functions under mod_qn
|
||||
existing_class_names = {c.short_name for c in ms.classes}
|
||||
existing_func_names = {f.short_name for f in ms.functions}
|
||||
for c in target.classes:
|
||||
if c.short_name in existing_class_names:
|
||||
continue
|
||||
ms.classes.append(StubClass(
|
||||
qualified_name=f"{mod_qn}.{c.short_name}",
|
||||
short_name=c.short_name,
|
||||
methods=list(c.methods),
|
||||
bases=list(c.bases),
|
||||
))
|
||||
existing_class_names.add(c.short_name)
|
||||
changed = True
|
||||
for f in target.functions:
|
||||
if f.short_name in existing_func_names:
|
||||
continue
|
||||
ms.functions.append(StubFunction(
|
||||
qualified_name=f"{mod_qn}.{f.short_name}",
|
||||
short_name=f.short_name,
|
||||
module_qn=mod_qn,
|
||||
))
|
||||
existing_func_names.add(f.short_name)
|
||||
changed = True
|
||||
|
||||
|
||||
def iter_module_stubs(stdlib_root: Path) -> Iterable[ModuleStubs]:
|
||||
modules, star_imports = collect_all_stubs(stdlib_root)
|
||||
resolve_reexports(modules, star_imports)
|
||||
for mod_qn in sorted(modules.keys()):
|
||||
yield modules[mod_qn]
|
||||
|
||||
|
||||
C_HEADER = """\
|
||||
// AUTO-GENERATED by scripts/gen-py-stdlib.py — DO NOT EDIT
|
||||
// Python stdlib type information for LSP type resolver.
|
||||
//
|
||||
// Source: typeshed commit a7912d521e16ff63caf7a8b64b9072542be36777
|
||||
// Module allowlist: see ALLOWED_MODULES in scripts/gen-py-stdlib.py
|
||||
//
|
||||
// v1 simplifications (per PYTHON_LSP_PLAN.md Phase 10):
|
||||
// - overload stacks collapse to first signature
|
||||
// - ParamSpec / TypeVarTuple / Concatenate skipped
|
||||
// - version guards flattened (union of all branches)
|
||||
// - per-symbol min/max version guards not yet emitted
|
||||
|
||||
#include "../type_rep.h"
|
||||
#include "../type_registry.h"
|
||||
#include <string.h>
|
||||
|
||||
#define CBM_PYTHON_STDLIB_GENERATED 1
|
||||
|
||||
void cbm_python_stdlib_register(CBMTypeRegistry* reg, CBMArena* arena) {
|
||||
if (!reg || !arena) return;
|
||||
|
||||
CBMRegisteredType rt;
|
||||
CBMRegisteredFunc rf;
|
||||
"""
|
||||
|
||||
C_FOOTER = """\
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def c_escape(s: str) -> str:
|
||||
return s.replace("\\", "\\\\").replace("\"", "\\\"")
|
||||
|
||||
|
||||
def emit(stubs: list[ModuleStubs], output_path: Path) -> None:
|
||||
method_table_id = 0
|
||||
out_lines: list[str] = []
|
||||
|
||||
for ms in stubs:
|
||||
if not ms.classes and not ms.functions:
|
||||
continue
|
||||
out_lines.append(f"\n /* ===== module: {ms.module_qn} ===== */\n")
|
||||
|
||||
for cls in ms.classes:
|
||||
method_array_name = None
|
||||
if cls.methods:
|
||||
method_table_id += 1
|
||||
method_array_name = f"py_methods_{method_table_id}"
|
||||
methods_c = ", ".join(f"\"{c_escape(m)}\"" for m in cls.methods)
|
||||
out_lines.append(
|
||||
f" static const char* {method_array_name}[] = {{ {methods_c}, NULL }};\n"
|
||||
)
|
||||
out_lines.append(" memset(&rt, 0, sizeof(rt));\n")
|
||||
out_lines.append(
|
||||
f" rt.qualified_name = \"{c_escape(cls.qualified_name)}\";\n"
|
||||
)
|
||||
out_lines.append(
|
||||
f" rt.short_name = \"{c_escape(cls.short_name)}\";\n"
|
||||
)
|
||||
if method_array_name:
|
||||
out_lines.append(f" rt.method_names = {method_array_name};\n")
|
||||
if cls.bases:
|
||||
method_table_id += 1
|
||||
bases_array = f"py_bases_{method_table_id}"
|
||||
bases_c = ", ".join(f"\"{c_escape(b)}\"" for b in cls.bases)
|
||||
out_lines.append(
|
||||
f" static const char* {bases_array}[] = {{ {bases_c}, NULL }};\n"
|
||||
)
|
||||
out_lines.append(f" rt.embedded_types = {bases_array};\n")
|
||||
out_lines.append(" cbm_registry_add_type(reg, rt);\n")
|
||||
|
||||
# Also emit a registered method per class.method so registry
|
||||
# lookup_method(class_qn, name) works.
|
||||
for m in cls.methods:
|
||||
out_lines.append(" memset(&rf, 0, sizeof(rf));\n")
|
||||
out_lines.append(
|
||||
f" rf.qualified_name = \"{c_escape(cls.qualified_name)}.{c_escape(m)}\";\n"
|
||||
)
|
||||
out_lines.append(f" rf.short_name = \"{c_escape(m)}\";\n")
|
||||
out_lines.append(
|
||||
f" rf.receiver_type = \"{c_escape(cls.qualified_name)}\";\n"
|
||||
)
|
||||
out_lines.append(" cbm_registry_add_func(reg, rf);\n")
|
||||
|
||||
for fn in ms.functions:
|
||||
out_lines.append(" memset(&rf, 0, sizeof(rf));\n")
|
||||
out_lines.append(
|
||||
f" rf.qualified_name = \"{c_escape(fn.qualified_name)}\";\n"
|
||||
)
|
||||
out_lines.append(f" rf.short_name = \"{c_escape(fn.short_name)}\";\n")
|
||||
out_lines.append(" cbm_registry_add_func(reg, rf);\n")
|
||||
|
||||
output_path.write_text(C_HEADER + "".join(out_lines) + C_FOOTER, encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"stdlib_root",
|
||||
nargs="?",
|
||||
default="/tmp/python-lsp-references/typeshed/stdlib",
|
||||
)
|
||||
parser.add_argument(
|
||||
"output",
|
||||
nargs="?",
|
||||
default="internal/cbm/lsp/generated/python_stdlib_data.c",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path(args.stdlib_root).resolve()
|
||||
if not root.is_dir():
|
||||
print(f"error: {root} is not a directory", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
stubs = list(iter_module_stubs(root))
|
||||
out_path = Path(args.output).resolve()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
emit(stubs, out_path)
|
||||
|
||||
n_classes = sum(len(s.classes) for s in stubs)
|
||||
n_methods = sum(sum(len(c.methods) for c in s.classes) for s in stubs)
|
||||
n_funcs = sum(len(s.functions) for s in stubs)
|
||||
print(
|
||||
f"wrote {out_path}: {len(stubs)} modules, "
|
||||
f"{n_classes} classes ({n_methods} methods), {n_funcs} free functions"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Generates the third-party notices bundle shipped inside every release
|
||||
# archive: THIRD_PARTY.md + the grammar provenance manifest + the verbatim
|
||||
# license/notice text of every vendored component (both vendored trees).
|
||||
# Deterministic output (sorted file order).
|
||||
#
|
||||
# Usage: scripts/gen-third-party-notices.sh [output-path]
|
||||
# default output: build/THIRD_PARTY_NOTICES.md
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
OUT="${1:-$ROOT/build/THIRD_PARTY_NOTICES.md}"
|
||||
mkdir -p "$(dirname "$OUT")"
|
||||
|
||||
{
|
||||
echo "# Third-Party Notices"
|
||||
echo
|
||||
echo "This file accompanies the codebase-memory-mcp binary distribution."
|
||||
echo "It aggregates THIRD_PARTY.md, the vendored grammar provenance"
|
||||
echo "manifest, and the verbatim license / notice texts of every vendored"
|
||||
echo "component, satisfying binary-redistribution notice requirements"
|
||||
echo "(MIT, BSD, Apache-2.0)."
|
||||
echo
|
||||
echo "---"
|
||||
echo
|
||||
cat "$ROOT/THIRD_PARTY.md"
|
||||
echo
|
||||
echo "---"
|
||||
echo
|
||||
cat "$ROOT/internal/cbm/vendored/grammars/MANIFEST.md"
|
||||
|
||||
find "$ROOT/vendored" "$ROOT/internal/cbm/vendored" -type f \
|
||||
\( -iname 'LICENSE*' -o -iname 'COPYING*' -o -iname 'NOTICE*' -o -iname 'UNLICENSE*' \) \
|
||||
| LC_ALL=C sort | while IFS= read -r f; do
|
||||
rel="${f#"$ROOT"/}"
|
||||
echo
|
||||
echo "==============================================================="
|
||||
echo " $rel"
|
||||
echo "==============================================================="
|
||||
echo
|
||||
cat "$f"
|
||||
done
|
||||
|
||||
# With-ui packaging path: node_modules exists (make frontend ran npm ci),
|
||||
# so append the license texts of the npm packages compiled into the UI
|
||||
# bundle. Standard binaries carry no bundle — the section is skipped.
|
||||
if [ -d "$ROOT/graph-ui/node_modules" ]; then
|
||||
echo
|
||||
echo "---"
|
||||
echo
|
||||
python3 "$ROOT/scripts/gen-ui-licenses.py" "$ROOT/graph-ui"
|
||||
fi
|
||||
} > "$OUT"
|
||||
|
||||
BYTES=$(wc -c < "$OUT" | tr -d ' ')
|
||||
echo "wrote $OUT (${BYTES} bytes)"
|
||||
Executable
+113
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Emit a markdown license appendix for the embedded graph-UI bundle.
|
||||
|
||||
Usage: gen-ui-licenses.py <graph-ui-dir>
|
||||
|
||||
Walks the PRODUCTION dependency tree (npm ls --omit=dev) — the packages
|
||||
whose code is compiled into the UI bundle — and prints, for each unique
|
||||
package, its declared license and the verbatim license file text found in
|
||||
node_modules. Build-time tooling (vite, tailwind, lightningcss, ...) is
|
||||
excluded: its code does not ship in the bundle.
|
||||
|
||||
Called by gen-third-party-notices.sh when node_modules is present (the
|
||||
with-ui packaging path). Output is deterministic (sorted by name@version).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def collect(ui_dir, dep_tree, out, excluded):
|
||||
for name, info in (dep_tree or {}).items():
|
||||
version = info.get("version")
|
||||
if not version:
|
||||
continue # unmet optional peer — not installed, nothing shipped
|
||||
pkg_dir = os.path.join(ui_dir, "node_modules", *name.split("/"))
|
||||
try:
|
||||
with open(os.path.join(pkg_dir, "package.json"), encoding="utf-8") as fh:
|
||||
meta = json.load(fh)
|
||||
except (OSError, ValueError):
|
||||
continue # not physically installed — nothing shipped
|
||||
# Platform-restricted packages (os/cpu fields) are native build
|
||||
# tooling (esbuild/rollup/lightningcss/oxide binaries). They cannot
|
||||
# be part of a browser bundle — exclude them and do not recurse into
|
||||
# their dependency subtrees (wasm runtime shims etc.).
|
||||
if meta.get("os") or meta.get("cpu"):
|
||||
excluded.add((name, version))
|
||||
continue
|
||||
out.add((name, version))
|
||||
collect(ui_dir, info.get("dependencies"), out, excluded)
|
||||
|
||||
|
||||
def license_text(pkg_dir):
|
||||
if not os.path.isdir(pkg_dir):
|
||||
return None
|
||||
for fname in sorted(os.listdir(pkg_dir)):
|
||||
if fname.upper().startswith(("LICENSE", "LICENCE", "COPYING", "NOTICE")):
|
||||
try:
|
||||
with open(os.path.join(pkg_dir, fname), encoding="utf-8", errors="replace") as fh:
|
||||
return fh.read().strip()
|
||||
except OSError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def declared_license(pkg_dir):
|
||||
try:
|
||||
with open(os.path.join(pkg_dir, "package.json"), encoding="utf-8") as fh:
|
||||
d = json.load(fh)
|
||||
except (OSError, ValueError):
|
||||
return "UNKNOWN"
|
||||
lic = d.get("license", "UNKNOWN")
|
||||
if isinstance(lic, dict):
|
||||
lic = lic.get("type", "UNKNOWN")
|
||||
return str(lic)
|
||||
|
||||
|
||||
def main():
|
||||
ui_dir = sys.argv[1]
|
||||
# shell=True so the npm shim resolves on every platform (npm.cmd on
|
||||
# Windows is not found by bare-name exec). Constant command, no injection.
|
||||
ls = subprocess.run(
|
||||
"npm ls --omit=dev --all --json",
|
||||
shell=True, cwd=ui_dir, capture_output=True, text=True, check=False,
|
||||
)
|
||||
tree = json.loads(ls.stdout or "{}")
|
||||
pkgs = set()
|
||||
excluded = set()
|
||||
collect(ui_dir, tree.get("dependencies"), pkgs, excluded)
|
||||
|
||||
print("## Embedded Graph UI — bundled npm packages")
|
||||
print()
|
||||
print("The `-ui` binaries embed a compiled frontend bundle. The packages")
|
||||
print("below are its production dependency tree; their license texts are")
|
||||
print("reproduced verbatim from the packages as installed at build time.")
|
||||
print()
|
||||
|
||||
for name, version in sorted(pkgs):
|
||||
pkg_dir = os.path.join(ui_dir, "node_modules", *name.split("/"))
|
||||
lic = declared_license(pkg_dir)
|
||||
text = license_text(pkg_dir)
|
||||
print(f"### {name}@{version} — {lic}")
|
||||
print()
|
||||
if text:
|
||||
print(text)
|
||||
else:
|
||||
print(f"(no license file shipped in the package; declared license: {lic})")
|
||||
print()
|
||||
|
||||
if excluded:
|
||||
print("### Platform-specific build tooling (not part of the bundle)")
|
||||
print()
|
||||
print("Resolved in the dependency tree but excluded above: these are")
|
||||
print("native per-platform build binaries whose code does not ship in")
|
||||
print("the browser bundle.")
|
||||
print()
|
||||
for name, version in sorted(excluded):
|
||||
print(f"- {name}@{version}")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate boilerplate code for new tree-sitter language support.
|
||||
|
||||
Reads scripts/new-languages.json and generates:
|
||||
1. Grammar wrapper .c files (written directly)
|
||||
2. Enum entries for cbm.h
|
||||
3. Lang spec entries for lang_specs.c (designated initializer + factory)
|
||||
4. Extension/filename/name entries for language.c
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PROJECT_DIR = os.path.dirname(SCRIPT_DIR)
|
||||
MANIFEST = os.path.join(SCRIPT_DIR, "new-languages.json")
|
||||
GRAMMAR_DIR = os.path.join(PROJECT_DIR, "internal", "cbm")
|
||||
|
||||
|
||||
def main():
|
||||
with open(MANIFEST) as f:
|
||||
langs = json.load(f)
|
||||
|
||||
mode = sys.argv[1] if len(sys.argv) > 1 else "all"
|
||||
|
||||
if mode in ("all", "wrappers"):
|
||||
generate_wrappers(langs)
|
||||
if mode in ("all", "enum"):
|
||||
generate_enum(langs)
|
||||
if mode in ("all", "specs"):
|
||||
generate_specs(langs)
|
||||
if mode in ("all", "language"):
|
||||
generate_language_c(langs)
|
||||
|
||||
|
||||
def generate_wrappers(langs):
|
||||
"""Create grammar_<name>.c wrapper files."""
|
||||
print("=== Grammar Wrapper Files ===")
|
||||
created = 0
|
||||
for lang in langs:
|
||||
path = os.path.join(GRAMMAR_DIR, f"grammar_{lang['name']}.c")
|
||||
if os.path.exists(path):
|
||||
continue
|
||||
lines = [
|
||||
f"// Vendored tree-sitter grammar: {lang['name']}",
|
||||
"// Each grammar compiled as separate unit (conflicting static symbols).",
|
||||
f"#include \"vendored/grammars/{lang['name']}/parser.c\"",
|
||||
]
|
||||
if lang["has_scanner"]:
|
||||
lines.append(
|
||||
f"#include \"vendored/grammars/{lang['name']}/scanner.c\""
|
||||
)
|
||||
with open(path, "w") as f:
|
||||
f.write("\n".join(lines) + "\n")
|
||||
created += 1
|
||||
print(f" Created {created} wrapper files")
|
||||
|
||||
|
||||
def generate_enum(langs):
|
||||
"""Print enum entries for cbm.h."""
|
||||
print("\n=== Enum Entries (paste into cbm.h before CBM_LANG_KUSTOMIZE) ===")
|
||||
for lang in langs:
|
||||
print(f" CBM_LANG_{lang['enum']},")
|
||||
|
||||
|
||||
def generate_specs(langs):
|
||||
"""Print lang spec entries for lang_specs.c."""
|
||||
print("\n=== Extern Declarations (paste at top of lang_specs.c) ===")
|
||||
for lang in langs:
|
||||
print(f"extern const TSLanguage *{lang['ts_func']}(void);")
|
||||
|
||||
print("\n=== Module Type Arrays (paste before spec table) ===")
|
||||
for lang in langs:
|
||||
arr = f"{lang['name']}_module_types"
|
||||
print(
|
||||
f'static const char *{arr}[] = {{"{lang["module_root"]}", NULL}};'
|
||||
)
|
||||
|
||||
print("\n=== Spec Table Entries (paste into lang_specs[]) ===")
|
||||
for lang in langs:
|
||||
mod = f"{lang['name']}_module_types"
|
||||
print(f" // CBM_LANG_{lang['enum']}")
|
||||
print(
|
||||
f" [CBM_LANG_{lang['enum']}] = {{CBM_LANG_{lang['enum']}, "
|
||||
f"empty_types, empty_types, empty_types, {mod}, "
|
||||
f"empty_types, empty_types, empty_types, empty_types, "
|
||||
f"empty_types, empty_types, empty_types, NULL, empty_types, "
|
||||
f"NULL, NULL, {lang['ts_func']}}},"
|
||||
)
|
||||
print()
|
||||
|
||||
|
||||
def generate_language_c(langs):
|
||||
"""Print extension table and name entries for language.c."""
|
||||
print("\n=== EXT_TABLE Entries (paste into language.c, sorted by ext) ===")
|
||||
ext_entries = []
|
||||
for lang in langs:
|
||||
for ext in lang["extensions"]:
|
||||
ext_entries.append((ext, lang["enum"], lang["display"]))
|
||||
for ext, enum, display in sorted(ext_entries, key=lambda x: x[0].lower()):
|
||||
print(f' /* {display} */')
|
||||
print(f' {{"{ext}", CBM_LANG_{enum}}},')
|
||||
print()
|
||||
|
||||
print("\n=== FILENAME_TABLE Entries ===")
|
||||
for lang in langs:
|
||||
for fn in lang["filenames"]:
|
||||
print(f' {{"{fn}", CBM_LANG_{lang["enum"]}}},')
|
||||
|
||||
print("\n=== LANG_NAMES Entries ===")
|
||||
for lang in langs:
|
||||
print(
|
||||
f' [CBM_LANG_{lang["enum"]}] = "{lang["display"]}",'
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# commit-msg hook — STRICT DCO enforcement at commit time.
|
||||
# Every commit must carry a Signed-off-by trailer (git commit -s).
|
||||
# Install with: scripts/install-git-hooks.sh
|
||||
|
||||
if grep -qE '^Signed-off-by: .+ <.+@.+>' "$1"; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "" >&2
|
||||
echo "COMMIT REJECTED: missing Signed-off-by trailer (Developer Certificate of Origin)." >&2
|
||||
echo "" >&2
|
||||
echo " Sign your commit: git commit -s" >&2
|
||||
echo " Fix the last commit: git commit --amend -s" >&2
|
||||
echo "" >&2
|
||||
echo "The sign-off certifies you have the right to submit this code under the" >&2
|
||||
echo "project's MIT license — see the DCO file and CONTRIBUTING.md." >&2
|
||||
exit 1
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Pre-commit hook: linters + security audit + build + tests.
|
||||
#
|
||||
# Activated automatically via scripts/setup.sh or manually:
|
||||
# git config core.hooksPath scripts/hooks
|
||||
|
||||
echo "pre-commit: running all linters in parallel..."
|
||||
make -j3 -f Makefile.cbm lint
|
||||
|
||||
echo "pre-commit: security audit (source-level)..."
|
||||
scripts/security-audit.sh
|
||||
|
||||
echo "pre-commit: building and running tests..."
|
||||
make -j$(sysctl -n hw.ncpu 2>/dev/null || nproc) -f Makefile.cbm test
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Install the repo's git hooks into .git/hooks (local, per-clone).
|
||||
# Currently: commit-msg (strict DCO sign-off enforcement).
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
HOOKS_DIR="$(git -C "$ROOT" rev-parse --git-path hooks)"
|
||||
|
||||
install -m 755 "$ROOT/scripts/git-hooks/commit-msg" "$HOOKS_DIR/commit-msg"
|
||||
echo "installed: $HOOKS_DIR/commit-msg (DCO sign-off required on every commit)"
|
||||
Executable
+109
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""License-gate check for the graph-UI npm production dependency tree.
|
||||
|
||||
Usage: license-gate-check-npm.py <graph-ui-dir> <license-policy.json>
|
||||
|
||||
Walks the production tree (the packages whose code is compiled into the UI
|
||||
bundle; platform-restricted native build tooling is excluded the same way
|
||||
gen-ui-licenses.py excludes it) and fails if ANY package's license is not
|
||||
on the policy allow-list. Packages with no declared license fall back to
|
||||
the first line of their shipped license file; if that cannot be resolved
|
||||
either, the gate fails — unknown is not allowed.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
FILE_HEADER_MAP = {
|
||||
"mit license": "MIT",
|
||||
"the mit license": "MIT",
|
||||
"the mit license (mit)": "MIT",
|
||||
"apache license": "Apache-2.0",
|
||||
"isc license": "ISC",
|
||||
"bsd 2-clause license": "BSD-2-Clause",
|
||||
"bsd 3-clause license": "BSD-3-Clause",
|
||||
"the unlicense": "Unlicense",
|
||||
}
|
||||
|
||||
|
||||
def license_from_file(pkg_dir):
|
||||
for fname in sorted(os.listdir(pkg_dir)):
|
||||
if fname.upper().startswith(("LICENSE", "LICENCE", "COPYING", "UNLICENSE")):
|
||||
try:
|
||||
with open(os.path.join(pkg_dir, fname), encoding="utf-8",
|
||||
errors="replace") as fh:
|
||||
first = fh.readline().strip().lower()
|
||||
return FILE_HEADER_MAP.get(first)
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def collect(ui_dir, dep_tree, out):
|
||||
for name, info in (dep_tree or {}).items():
|
||||
version = info.get("version")
|
||||
if not version:
|
||||
continue
|
||||
pkg_dir = os.path.join(ui_dir, "node_modules", *name.split("/"))
|
||||
try:
|
||||
with open(os.path.join(pkg_dir, "package.json"), encoding="utf-8") as fh:
|
||||
meta = json.load(fh)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if meta.get("os") or meta.get("cpu"):
|
||||
continue # platform-restricted native build tooling, not bundled
|
||||
lic = meta.get("license", "")
|
||||
if isinstance(lic, dict):
|
||||
lic = lic.get("type", "")
|
||||
out[(name, version)] = (str(lic or ""), pkg_dir)
|
||||
collect(ui_dir, info.get("dependencies"), out)
|
||||
|
||||
|
||||
def main():
|
||||
ui_dir, policy_path = sys.argv[1], sys.argv[2]
|
||||
with open(policy_path) as fh:
|
||||
policy = json.load(fh)
|
||||
allowed = {x.lower() for x in policy["allowed_spdx_ids"]}
|
||||
ignored_pkgs = set(policy.get("ignored_npm_packages", []))
|
||||
skip_tokens = {"and", "or", "with", ""}
|
||||
|
||||
# shell=True so the npm shim resolves on every platform (npm.cmd on
|
||||
# Windows is not found by bare-name exec). Constant command, no injection.
|
||||
ls = subprocess.run("npm ls --omit=dev --all --json",
|
||||
shell=True, cwd=ui_dir, capture_output=True, text=True, check=False)
|
||||
tree = json.loads(ls.stdout or "{}")
|
||||
pkgs = {}
|
||||
collect(ui_dir, tree.get("dependencies"), pkgs)
|
||||
if not pkgs:
|
||||
print("FAIL: npm production tree resolved to zero packages — "
|
||||
"is node_modules installed?")
|
||||
sys.exit(1)
|
||||
|
||||
violations = []
|
||||
for (name, version), (lic, pkg_dir) in sorted(pkgs.items()):
|
||||
if name in ignored_pkgs:
|
||||
continue
|
||||
if not lic:
|
||||
lic = license_from_file(pkg_dir) or ""
|
||||
if not lic:
|
||||
violations.append((f"{name}@{version}", "no resolvable license"))
|
||||
continue
|
||||
for tok in re.split(r"[\s()]+", lic):
|
||||
if tok.lower() in skip_tokens:
|
||||
continue
|
||||
if tok.lower() not in allowed:
|
||||
violations.append((f"{name}@{version}", lic))
|
||||
break
|
||||
|
||||
if violations:
|
||||
print("BLOCKED: %d UI package(s) outside the license allow-list:" % len(violations))
|
||||
for pkg, lic in violations[:25]:
|
||||
print(f" {pkg}: {lic}")
|
||||
sys.exit(1)
|
||||
print(f"OK: {len(pkgs)} bundled npm packages, all allow-listed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""License-gate policy check over a ScanCode Toolkit JSON scan.
|
||||
|
||||
Usage: license-gate-check.py <scan.json> <license-policy.json>
|
||||
|
||||
Fails (exit 1) if ANY scanned file carries a detected license expression
|
||||
containing an SPDX id outside the policy allow-list — one finding is enough.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
scan_path, policy_path = sys.argv[1], sys.argv[2]
|
||||
with open(policy_path) as fh:
|
||||
policy = json.load(fh)
|
||||
allowed = {x.lower() for x in policy["allowed_spdx_ids"]}
|
||||
ignored_paths = tuple(policy.get("ignored_paths", []))
|
||||
skip_tokens = {"and", "or", "with", ""}
|
||||
|
||||
with open(scan_path) as fh:
|
||||
scan = json.load(fh)
|
||||
|
||||
violations = []
|
||||
checked = 0
|
||||
for f in scan.get("files", []):
|
||||
path = f.get("path", "?")
|
||||
if f.get("type") != "file":
|
||||
continue
|
||||
if ignored_paths and path.startswith(ignored_paths):
|
||||
continue
|
||||
expr = f.get("detected_license_expression_spdx")
|
||||
if not expr:
|
||||
continue
|
||||
checked += 1
|
||||
for tok in re.split(r"[\s()]+", expr):
|
||||
if tok.lower() in skip_tokens:
|
||||
continue
|
||||
if tok.lower() not in allowed:
|
||||
violations.append((path, expr, tok))
|
||||
break
|
||||
|
||||
if violations:
|
||||
print("BLOCKED: %d file(s) with non-allow-listed license detections:" % len(violations))
|
||||
for path, expr, tok in violations[:25]:
|
||||
print(" %s: '%s' (offending id: %s)" % (path, expr, tok))
|
||||
sys.exit(1)
|
||||
print("OK: %d detection(s), all allow-listed" % checked)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+124
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# CI license-compliance gate (runs in the security workflow for both dry
|
||||
# runs and releases). Two layers:
|
||||
# 1) Structural: every vendored component directory must physically carry
|
||||
# a license file.
|
||||
# 2) ScanCode Toolkit license detection over every vendored license text
|
||||
# and all first-party sources; ONE detection outside
|
||||
# scripts/license-policy.json fails the gate (see license-gate-check.py).
|
||||
#
|
||||
# Grammar parser bodies (generated parser.c, no headers) are excluded from
|
||||
# the ScanCode pass for runtime; their per-directory license files ARE
|
||||
# scanned, and layer 1 guarantees each grammar carries one.
|
||||
#
|
||||
# Requires: scancode (pipx install scancode-toolkit), python3.
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# --selftest: plant a violation and assert the structural layer catches it.
|
||||
# Run in CI before the real gate so a silently-broken gate cannot pass.
|
||||
if [ "${1:-}" = "--selftest" ]; then
|
||||
TESTDIR="$ROOT/vendored/.gate-selftest-$$"
|
||||
mkdir -p "$TESTDIR"
|
||||
echo "int x;" > "$TESTDIR/x.c"
|
||||
# The child gate exits 1 on the planted violation (that is the point) —
|
||||
# capture its output first so pipefail cannot mask the grep result.
|
||||
GATE_OUT="$("$0" 2>/dev/null || true)"
|
||||
rm -rf "$TESTDIR"
|
||||
if printf '%s' "$GATE_OUT" | grep -q "BLOCKED: vendored code in .*gate-selftest"; then
|
||||
echo "OK: gate self-test — planted violation was detected"
|
||||
exit 0
|
||||
fi
|
||||
echo "FAIL: gate self-test — planted unlicensed file was NOT detected"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== License gate 1/2: structural coverage ==="
|
||||
# Rule: every directory under either vendored tree that contains source or
|
||||
# data files must have a license file in itself or an ancestor directory
|
||||
# WITHIN the vendored tree. No hardcoded component list — newly vendored
|
||||
# code with no license is flagged immediately.
|
||||
MISS=0
|
||||
has_license_dir() {
|
||||
ls "$1" 2>/dev/null | grep -qiE '^(LICENSE|LICENCE|COPYING|UNLICENSE|NOTICE)'
|
||||
}
|
||||
for root in vendored internal/cbm/vendored; do
|
||||
find "$root" -type f \
|
||||
\( -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp' \
|
||||
-o -name '*.S' -o -name '*.bin' -o -name '*.txt' \) \
|
||||
! -iname 'LICENSE*' ! -iname 'COPYING*' ! -iname 'NOTICE*' \
|
||||
-exec dirname {} \; | LC_ALL=C sort -u | while IFS= read -r d; do
|
||||
cur="$d"
|
||||
covered=1
|
||||
while :; do
|
||||
if has_license_dir "$cur"; then
|
||||
covered=0
|
||||
break
|
||||
fi
|
||||
[ "$cur" = "$root" ] && break
|
||||
cur="$(dirname "$cur")"
|
||||
done
|
||||
if [ $covered -ne 0 ]; then
|
||||
echo "BLOCKED: vendored code in $d has no license file in itself or any ancestor within $root/"
|
||||
fi
|
||||
done > /tmp/license-gate-structural.$$
|
||||
if [ -s /tmp/license-gate-structural.$$ ]; then
|
||||
cat /tmp/license-gate-structural.$$
|
||||
MISS=1
|
||||
fi
|
||||
rm -f /tmp/license-gate-structural.$$
|
||||
done
|
||||
if [ $MISS -ne 0 ]; then
|
||||
echo "=== LICENSE GATE FAILED (structural) ==="
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: every vendored source directory is covered by a license file"
|
||||
|
||||
echo "=== License gate 2/2: ScanCode detection ==="
|
||||
if ! command -v scancode &>/dev/null; then
|
||||
echo "FAIL: scancode not installed (pipx install scancode-toolkit)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STAGE="$(mktemp -d)"
|
||||
trap 'rm -rf "$STAGE"' EXIT
|
||||
|
||||
# Inputs: all vendored license/notice texts + all first-party sources.
|
||||
find vendored internal/cbm/vendored -type f \
|
||||
\( -iname 'LICENSE*' -o -iname 'COPYING*' -o -iname 'NOTICE*' -o -iname 'UNLICENSE*' \) \
|
||||
> "$STAGE/files.txt"
|
||||
find src pkg scripts -type f \
|
||||
\( -name '*.c' -o -name '*.h' -o -name '*.sh' -o -name '*.js' \
|
||||
-o -name '*.py' -o -name '*.rb' -o -name '*.toml' -o -name '*.json' \) \
|
||||
>> "$STAGE/files.txt"
|
||||
|
||||
mkdir -p "$STAGE/tree"
|
||||
tar cf - -T "$STAGE/files.txt" | tar xf - -C "$STAGE/tree"
|
||||
|
||||
scancode --license --quiet --processes 2 --json-pp "$STAGE/scan.json" "$STAGE/tree" \
|
||||
> "$STAGE/scancode.log" 2>&1 || {
|
||||
echo "FAIL: scancode run failed:"
|
||||
tail -20 "$STAGE/scancode.log"
|
||||
exit 1
|
||||
}
|
||||
|
||||
python3 scripts/license-gate-check.py "$STAGE/scan.json" scripts/license-policy.json
|
||||
|
||||
echo "=== License gate 3/3: UI npm production tree ==="
|
||||
# The -ui binaries embed the compiled frontend bundle; its production
|
||||
# dependency tree must be allow-listed too. --ignore-scripts: no dependency
|
||||
# postinstall code runs inside the security job.
|
||||
if command -v npm &>/dev/null && [ -f graph-ui/package-lock.json ]; then
|
||||
if [ ! -d graph-ui/node_modules ]; then
|
||||
(cd graph-ui && npm ci --ignore-scripts --silent)
|
||||
fi
|
||||
python3 scripts/license-gate-check-npm.py graph-ui scripts/license-policy.json
|
||||
else
|
||||
echo "FAIL: npm or graph-ui/package-lock.json unavailable — UI tree unchecked"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== License gate passed ==="
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"_comment": "License allow-list for the CI license gate (scripts/license-gate.sh). A single detected SPDX id outside this list fails the pipeline. Additions to this list are deliberate, reviewed changes \u2014 never add copyleft (GPL/LGPL/AGPL/EPL/SSPL) without a maintainer decision.",
|
||||
"allowed_spdx_ids": [
|
||||
"MIT",
|
||||
"MIT-0",
|
||||
"Apache-2.0",
|
||||
"BSD-2-Clause",
|
||||
"BSD-3-Clause",
|
||||
"0BSD",
|
||||
"ISC",
|
||||
"Zlib",
|
||||
"Unlicense",
|
||||
"CC0-1.0",
|
||||
"blessing",
|
||||
"LLVM-exception",
|
||||
"LicenseRef-scancode-public-domain",
|
||||
"LicenseRef-scancode-public-domain-disclaimer"
|
||||
],
|
||||
"_ignored_paths_comment": "Path prefixes (relative to the staged scan tree) excluded from the gate. Use ONLY for documented false positives. Justifications: the license tooling itself (gate scripts + this policy file) necessarily names prohibited licenses; gen-third-party-notices.sh echoes license terminology; audit-license-provenance.py names licenses in its verdict maps; src/discover/discover.c contains a license-FILENAME classification list (LICENSE-MIT, LICENSE-APACHE, ...) for file discovery, which ScanCode reads as license references.",
|
||||
"ignored_paths": [
|
||||
"tree/scripts/license-policy.json",
|
||||
"tree/scripts/license-gate-check.py",
|
||||
"tree/scripts/license-gate.sh",
|
||||
"tree/scripts/gen-third-party-notices.sh",
|
||||
"tree/src/discover/discover.c",
|
||||
"tree/scripts/audit-license-provenance.py"
|
||||
]
|
||||
}
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# lint.sh — Run all linters (clang-tidy + cppcheck + clang-format).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/lint.sh # All 3 linters
|
||||
# scripts/lint.sh CLANG_FORMAT=clang-format-20 # Override formatter
|
||||
# scripts/lint.sh --ci # CI mode (skip clang-tidy)
|
||||
#
|
||||
# This script is the SINGLE source of truth for linting.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# shellcheck source=env.sh
|
||||
source "$ROOT/scripts/env.sh"
|
||||
|
||||
# Check for --ci flag (skip clang-tidy for platforms without it)
|
||||
CI_ONLY=false
|
||||
MAKE_ARGS=()
|
||||
for arg in "$@"; do
|
||||
if [ "$arg" = "--ci" ]; then
|
||||
CI_ONLY=true
|
||||
else
|
||||
MAKE_ARGS+=("$arg")
|
||||
fi
|
||||
done
|
||||
|
||||
print_env "lint.sh"
|
||||
|
||||
# No-skips policy: every test must pass or fail. The only tolerable skip is a
|
||||
# genuinely platform-specific test (SKIP_PLATFORM / #ifdef). Runs in both modes.
|
||||
echo "=== no-skips policy (tests pass or fail) ==="
|
||||
bash "$ROOT/scripts/check-no-test-skips.sh"
|
||||
|
||||
if $CI_ONLY; then
|
||||
echo "=== CI mode: cppcheck + clang-format ==="
|
||||
make -j2 -f Makefile.cbm lint-ci "${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}"
|
||||
echo "=== CI linters passed ==="
|
||||
else
|
||||
echo "=== Full lint: clang-tidy + cppcheck + clang-format ==="
|
||||
make -j3 -f Makefile.cbm lint "${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}"
|
||||
fi
|
||||
|
||||
echo "=== All linters passed ==="
|
||||
File diff suppressed because it is too large
Load Diff
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
# repro.sh — Build + run the cumulative BUG-REPRODUCTION suite (test-repro).
|
||||
#
|
||||
# Unlike test.sh (the gating suite, must be GREEN), this suite is RED by design:
|
||||
# every case reproduces an open bug. So we distinguish two outcomes:
|
||||
# - BUILD/LINK failure → real breakage → exit non-zero (fail the CI job).
|
||||
# - Test redness → EXPECTED → report the count, exit 0 (green board).
|
||||
#
|
||||
# Usage: scripts/repro.sh [CC=clang] [CXX=clang++] [--arch arm64|x86_64]
|
||||
set -uo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# --arch before sourcing env.sh (mirrors test.sh)
|
||||
prev_arg=""
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
arm64|x86_64) [[ "$prev_arg" == "--arch" ]] && export CBM_ARCH="$arg" ;;
|
||||
--arch=*) export CBM_ARCH="${arg#--arch=}" ;;
|
||||
esac
|
||||
prev_arg="$arg"
|
||||
done
|
||||
|
||||
# shellcheck source=env.sh
|
||||
source "$ROOT/scripts/env.sh"
|
||||
|
||||
MAKE_ARGS=""
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
CC=*|CXX=*) export "${arg?}" ;;
|
||||
--arch|--arch=*|arm64|x86_64) ;;
|
||||
*=*) MAKE_ARGS="$MAKE_ARGS $arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
print_env "repro.sh"
|
||||
verify_compiler "$CC"
|
||||
|
||||
OUT="$ROOT/repro-out.txt"
|
||||
# A RED reproduction fails its assertion and returns EARLY — before any cleanup —
|
||||
# so LeakSanitizer would flag benign harness leaks on every red store-level test
|
||||
# and abort. The board's signal is the FAIL rows, not leak-cleanliness (the leak
|
||||
# BUG #581 gets a dedicated RSS-growth test, not LSan). Disable leak detection
|
||||
# only; ASan's real checks (use-after-free, overflow) stay ON.
|
||||
export ASAN_OPTIONS="detect_leaks=0${ASAN_OPTIONS:+:$ASAN_OPTIONS}"
|
||||
|
||||
# test-repro both builds and runs the runner; tolerate its non-zero (red) exit.
|
||||
set +e
|
||||
make -j"$NPROC" -f Makefile.cbm test-repro $MAKE_ARGS 2>&1 | tee "$OUT"
|
||||
set -e
|
||||
|
||||
# The runner prints a "<N> passed[, <M> failed]" summary line only if it actually
|
||||
# ran. No summary line ⇒ the build/link failed ⇒ real breakage.
|
||||
if ! grep -qE '[0-9]+ passed' "$OUT"; then
|
||||
echo "::error::bug-repro runner did not execute — build or link failure"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
reproduced=$(grep -oE '[0-9]+ failed' "$OUT" | head -1 | grep -oE '[0-9]+' || echo 0)
|
||||
green=$(grep -oE '[0-9]+ passed' "$OUT" | head -1 | grep -oE '[0-9]+' || echo 0)
|
||||
|
||||
{
|
||||
echo "## Bug-reproduction board — ${OS:-$(uname -s)} ${ARCH:-}"
|
||||
echo ""
|
||||
echo "- **${reproduced}** open bug(s) still reproduced (RED — expected)"
|
||||
echo "- **${green}** case(s) PASSING — candidate-fixed → verify + close the issue + promote the guard to the gating suite"
|
||||
} >> "${GITHUB_STEP_SUMMARY:-/dev/stderr}"
|
||||
|
||||
echo "=== bug-repro board: ${reproduced} reproduced (RED), ${green} passing (candidate-fixed) ==="
|
||||
# Green board: the suite ran. Redness is the data, not a job failure.
|
||||
exit 0
|
||||
@@ -0,0 +1,59 @@
|
||||
# Security allow-list for dangerous function calls.
|
||||
# Format: file:function:justification
|
||||
# Lines starting with # are comments. Empty lines are ignored.
|
||||
# Any call to a listed function in a .c file under src/ that is NOT on this
|
||||
# list causes the security audit (scripts/security-audit.sh) to fail.
|
||||
|
||||
# ── Foundation: platform abstraction (defines cbm_popen wrapper + shell-free exec) ──
|
||||
src/foundation/compat_fs.c:popen:cbm_popen wrapper definition (POSIX)
|
||||
src/foundation/compat_fs.c:cbm_popen:cbm_popen function definition
|
||||
src/foundation/compat_fs.c:fork:cbm_exec_no_shell — fork+execvp for shell-free subprocess execution
|
||||
src/foundation/subprocess.c:fork:cbm_run_posix — fork+execv for the crash/hang-isolating index worker (supervisor primitive; child execs immediately, no code runs in the forked image)
|
||||
src/foundation/compat_fs.c:execvp:cbm_exec_no_shell — direct exec without shell interpretation
|
||||
|
||||
# ── CLI: update command (user-initiated, interactive) ──────────────────────
|
||||
src/cli/cli.c:cbm_popen:sha256 checksum verification (update cmd)
|
||||
src/cli/cli.c:cbm_popen:pgrep for kill_other_instances (hardcoded process name)
|
||||
src/cli/cli.c:popen:sha256 checksum computation via shasum
|
||||
|
||||
# ── Watcher: git status polling (repo paths validated via cbm_validate_shell_arg) ──
|
||||
src/watcher/watcher.c:system:git repo detection (is_git_repo)
|
||||
src/watcher/watcher.c:cbm_popen:git HEAD hash (git_head)
|
||||
src/watcher/watcher.c:cbm_popen:git working tree status (git_is_dirty)
|
||||
src/watcher/watcher.c:cbm_popen:git file count (git_file_count)
|
||||
src/watcher/watcher.c:popen:via cbm_popen wrapper calls
|
||||
|
||||
# ── Git context: git metadata resolution (repo paths validated via cbm_validate_shell_arg) ──
|
||||
src/git/git_context.c:cbm_popen:git rev-parse/symbolic-ref/merge-base metadata lookup
|
||||
src/git/git_context.c:popen:via cbm_popen wrapper call
|
||||
|
||||
# ── MCP server: search and change detection ────────────────────────────────
|
||||
src/mcp/mcp.c:cbm_popen:search_code via grep (pattern in temp file, path validated)
|
||||
src/mcp/mcp.c:cbm_popen:detect_changes via git diff (args validated)
|
||||
src/mcp/mcp.c:cbm_popen:git ls-files count for auto-index (session_root validated)
|
||||
src/mcp/mcp.c:cbm_popen:update check to api.github.com (hardcoded URL)
|
||||
src/mcp/mcp.c:popen:via cbm_popen wrapper calls
|
||||
|
||||
# ── Pipeline: git history parsing (git log) ────────────────────────────────
|
||||
src/pipeline/pass_githistory.c:cbm_popen:git log for file history (path validated)
|
||||
src/pipeline/pass_githistory.c:popen:via cbm_popen wrapper call
|
||||
|
||||
# ── Pipeline: artifact persistence (git HEAD hash, merge driver config) ────
|
||||
src/pipeline/artifact.c:cbm_popen:git rev-parse HEAD for artifact metadata (hardcoded cmd)
|
||||
src/pipeline/artifact.c:cbm_popen:git config merge.ours.driver for gitattributes (hardcoded cmd)
|
||||
src/pipeline/artifact.c:popen:via cbm_popen wrapper calls
|
||||
|
||||
# ── UI: HTTP server process management ─────────────────────────────────────
|
||||
src/ui/http_server.c:popen:ps process listing for metrics endpoint
|
||||
src/ui/http_server.c:fork:spawn indexing subprocess
|
||||
src/ui/http_server.c:execl:exec indexing binary in child process
|
||||
|
||||
# ── Allowed URLs ───────────────────────────────────────────────────────────
|
||||
# Format: URL:justification
|
||||
URL:https://api.github.com/repos/DeusData/codebase-memory-mcp/releases/latest:update check
|
||||
URL:https://github.com/DeusData/codebase-memory-mcp/releases/latest/download:binary download + checksums
|
||||
URL:https://github.com/DeusData/codebase-memory-mcp/releases/latest:version check via redirect header
|
||||
URL:http://127.0.0.1:UI server binding (localhost only)
|
||||
URL:https://www.sqlite.org/c3ref/c_checkpoint_full.html:sqlite WAL checkpoint API doc reference (comment only, not a network call)
|
||||
URL:https://github.com/DeusData/codebase-memory-mcp:project repository self-reference in update/star notice (src/mcp/mcp.c)
|
||||
URL:https://%s:GitHub blob-URL construction in src/ui/http_server.c cbm_ui_git_web_base — https-forces the repo web base for frontend deep-links (not a network call)
|
||||
Executable
+302
@@ -0,0 +1,302 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Layer 1: Static security audit — scans C source for dangerous calls.
|
||||
# Every occurrence must be on the checked-in allow-list.
|
||||
#
|
||||
# Usage: scripts/security-audit.sh
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
ALLOWLIST="$ROOT/scripts/security-allowlist.txt"
|
||||
|
||||
if [[ ! -f "$ALLOWLIST" ]]; then
|
||||
echo "FAIL: allow-list not found: $ALLOWLIST"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Use a flag file to communicate failures from subshells
|
||||
FAIL_FLAG=$(mktemp)
|
||||
echo "0" > "$FAIL_FLAG"
|
||||
trap 'rm -f "$FAIL_FLAG"' EXIT
|
||||
|
||||
fail() {
|
||||
echo "1" > "$FAIL_FLAG"
|
||||
}
|
||||
|
||||
# ── 1. Dangerous function calls ─────────────────────────────────
|
||||
|
||||
echo "=== Layer 1: Static Security Audit ==="
|
||||
echo ""
|
||||
echo "--- Scanning for dangerous function calls ---"
|
||||
|
||||
# For each file:function pair on the allow-list, the file is allowed to contain
|
||||
# that function. Any file:function NOT on the list causes failure.
|
||||
FUNC_LIST="system popen cbm_popen execl fork"
|
||||
|
||||
while IFS= read -r file; do
|
||||
relfile="${file#"$ROOT/"}"
|
||||
|
||||
for func in $FUNC_LIST; do
|
||||
# Build precise grep pattern to avoid substring matches:
|
||||
# 'popen(' must NOT match 'cbm_popen(' — use [^a-z] negative class
|
||||
case "$func" in
|
||||
cbm_popen) pattern="cbm_popen(" ;;
|
||||
popen) pattern="[^a-z]popen(" ;;
|
||||
system) pattern="[^a-z_]system(" ;;
|
||||
fork) pattern="[^a-z_]fork(" ;;
|
||||
*) pattern="[^a-z_]${func}(" ;;
|
||||
esac
|
||||
|
||||
# Grep for pattern, excluding comments and #define lines
|
||||
if grep -n "$pattern" "$file" 2>/dev/null | grep -v '^\s*//' | grep -v '^\s*\*' | grep -v '#define' > /dev/null 2>&1; then
|
||||
if ! grep -q "^${relfile}:${func}:" "$ALLOWLIST" 2>/dev/null; then
|
||||
echo "BLOCKED: ${relfile}: contains ${func}() — not on allow-list"
|
||||
grep -n "$pattern" "$file" 2>/dev/null | grep -v '^\s*//' | grep -v '^\s*\*' | grep -v '#define' | head -3 | sed 's/^/ /'
|
||||
fail
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done < <(find "$ROOT/src" -name '*.c' -type f | sort)
|
||||
|
||||
# ── 1b. Raw network calls (must not exist) ──────────────────────
|
||||
#
|
||||
# The graph-UI HTTP server (src/ui/httpd.c) is the one component that owns a
|
||||
# listening socket. It is first-party, binds 127.0.0.1 only, and is audited
|
||||
# separately by scripts/security-ui.sh (binding + CORS checks), so it is
|
||||
# exempt from this scan. No other source file may make raw network calls.
|
||||
|
||||
echo ""
|
||||
echo "--- Scanning for raw network calls (must not exist) ---"
|
||||
|
||||
NETWORK_FUNCS='[^a-z_]connect\(|[^a-z_]socket\(|[^a-z_]sendto\('
|
||||
if grep -rn -E "$NETWORK_FUNCS" "$ROOT/src/" --include='*.c' 2>/dev/null | grep -v '^\s*//' | grep -v '^\s*\*' | grep -v 'test' | grep -v 'src/ui/httpd.c'; then
|
||||
echo "BLOCKED: Raw network calls found in src/."
|
||||
fail
|
||||
else
|
||||
echo "OK: No raw network calls outside the graph-UI server."
|
||||
fi
|
||||
|
||||
# ── 2. Hardcoded URLs in string literals ─────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- Scanning for hardcoded URLs ---"
|
||||
|
||||
# Extract allowed URL prefixes from the allow-list
|
||||
# Format: URL:<url>:<justification>
|
||||
ALLOWED_URLS=()
|
||||
while IFS= read -r line; do
|
||||
[[ "$line" =~ ^#.*$ ]] && continue
|
||||
[[ -z "$line" ]] && continue
|
||||
if [[ "$line" =~ ^URL: ]]; then
|
||||
rest="${line#URL:}"
|
||||
# Extract URL (scheme://host/path) — stop at first colon that follows a non-slash
|
||||
if [[ "$rest" =~ ^(https?://[^[:space:]]+) ]]; then
|
||||
# The URL part extends until the justification separator
|
||||
# Use the fact that justifications follow the pattern ":word word"
|
||||
url_part="${BASH_REMATCH[1]}"
|
||||
# Remove trailing justification after last colon that precedes a space
|
||||
url_part="${url_part%%:[A-Za-z]*}"
|
||||
ALLOWED_URLS+=("$url_part")
|
||||
fi
|
||||
fi
|
||||
done < "$ALLOWLIST"
|
||||
|
||||
URL_OK=true
|
||||
|
||||
# Non-functional URL patterns to skip (comments, placeholders, comparisons, patterns)
|
||||
is_placeholder_url() {
|
||||
local url="$1"
|
||||
case "$url" in
|
||||
# Placeholder/example URLs in comments and code
|
||||
https://host/*|http://host/*) return 0 ;;
|
||||
https://...*|http://...*) return 0 ;;
|
||||
# Protocol prefix comparisons (strncmp, mg_match)
|
||||
http://) return 0 ;;
|
||||
https://) return 0 ;;
|
||||
# mg_match glob patterns with wildcards
|
||||
http://localhost:*) return 0 ;;
|
||||
http://127.0.0.1:*) return 0 ;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
|
||||
while IFS= read -r file; do
|
||||
relfile="${file#"$ROOT/"}"
|
||||
|
||||
# Find lines with URLs (excluding comments)
|
||||
while IFS= read -r match; do
|
||||
[[ -z "$match" ]] && continue
|
||||
|
||||
# Skip lines that are clearly comments (/* ... */ or // ...)
|
||||
line_content="${match#*:}" # Remove line number prefix
|
||||
# Skip if the URL appears only in a comment on this line
|
||||
if echo "$line_content" | grep -qE '^\s*/[/*]'; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Extract URLs using grep -oE (POSIX-compatible)
|
||||
while IFS= read -r url; do
|
||||
[[ -z "$url" ]] && continue
|
||||
|
||||
# Skip non-functional placeholder URLs
|
||||
if is_placeholder_url "$url"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
allowed=false
|
||||
for aurl in "${ALLOWED_URLS[@]+"${ALLOWED_URLS[@]}"}"; do
|
||||
if [[ "$url" == "$aurl"* ]]; then
|
||||
allowed=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if ! $allowed; then
|
||||
echo "BLOCKED: ${relfile}: URL not on allow-list: $url"
|
||||
fail
|
||||
URL_OK=false
|
||||
fi
|
||||
done < <(echo "$match" | grep -oE 'https?://[A-Za-z0-9._/~:@!$&()*+,;=?#%-]+' || true)
|
||||
done < <(grep -n 'https\?://' "$file" 2>/dev/null | grep -v '^\s*//' | grep -v '^\s*\*' || true)
|
||||
done < <(find "$ROOT/src" -name '*.c' -type f | sort)
|
||||
|
||||
if $URL_OK; then
|
||||
echo "OK: All URLs are on the allow-list."
|
||||
fi
|
||||
|
||||
# ── 3. File writes outside expected paths ────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- Scanning for unexpected file writes in src/ ---"
|
||||
|
||||
FOPEN_FOUND=false
|
||||
while IFS= read -r match; do
|
||||
[[ -z "$match" ]] && continue
|
||||
file=$(echo "$match" | cut -d: -f1)
|
||||
relfile="${file#"$ROOT/"}"
|
||||
case "$relfile" in
|
||||
src/cli/cli.c|src/store/store.c|src/pipeline/*.c|src/foundation/log.c|src/foundation/diagnostics.c|src/ui/http_server.c|src/ui/config.c|src/mcp/mcp.c)
|
||||
;; # Known safe (diagnostics.c: atomic .tmp+rename metrics dump to configured path)
|
||||
*)
|
||||
echo "REVIEW: ${match}"
|
||||
echo " -> Unexpected fopen(\"w\") in ${relfile}"
|
||||
FOPEN_FOUND=true
|
||||
;;
|
||||
esac
|
||||
done < <(grep -rn 'fopen.*"w' "$ROOT/src/" --include='*.c' 2>/dev/null | grep -v '/test' | grep -v '^\s*//' || true)
|
||||
|
||||
if ! $FOPEN_FOUND; then
|
||||
echo "OK: All file writes are in expected locations."
|
||||
fi
|
||||
|
||||
# ── 4. Time-bomb pattern detection ────────────────────────────────
|
||||
# Scan for time/clock/sleep near dangerous calls — could indicate
|
||||
# code that activates malicious behavior after a delay or date.
|
||||
|
||||
echo ""
|
||||
echo "--- Scanning for time-bomb patterns ---"
|
||||
|
||||
TIMEBOMB_FOUND=false
|
||||
while IFS= read -r file; do
|
||||
relfile="${file#"$ROOT/"}"
|
||||
|
||||
# Extract line numbers of dangerous calls
|
||||
DANGER_LINES=$(grep -n 'system(\|popen(\|cbm_popen(\|connect(\|execl(' "$file" 2>/dev/null \
|
||||
| grep -v '^\s*//' | grep -v '^\s*\*' | grep -v '#define' \
|
||||
| cut -d: -f1 || true)
|
||||
|
||||
[ -z "$DANGER_LINES" ] && continue
|
||||
|
||||
# For each dangerous call, check if time/clock/sleep appears within 10 lines
|
||||
for dline in $DANGER_LINES; do
|
||||
start=$((dline > 10 ? dline - 10 : 1))
|
||||
end=$((dline + 10))
|
||||
NEARBY=$(sed -n "${start},${end}p" "$file" 2>/dev/null \
|
||||
| grep -E 'time\(|clock\(|sleep\(|nanosleep\(|usleep\(' \
|
||||
| grep -v '^\s*//' | grep -v '^\s*\*' | grep -v 'timeout' | grep -v 'elapsed' || true)
|
||||
if [ -n "$NEARBY" ]; then
|
||||
echo "REVIEW: ${relfile}:${dline}: time-related call near dangerous function"
|
||||
echo " $NEARBY"
|
||||
TIMEBOMB_FOUND=true
|
||||
fi
|
||||
done
|
||||
done < <(find "$ROOT/src" -name '*.c' -type f | sort)
|
||||
|
||||
if ! $TIMEBOMB_FOUND; then
|
||||
echo "OK: No suspicious time-bomb patterns found."
|
||||
fi
|
||||
|
||||
# ── 5. MCP tool handler file read audit ──────────────────────────
|
||||
# The MCP server (mcp.c) handles tool calls that return data to the
|
||||
# client. A malicious PR could add file reads that exfiltrate sensitive
|
||||
# data (e.g., ~/.ssh/id_rsa) through the normal tool response channel.
|
||||
# Track all file-reading functions in mcp.c against an allow-list.
|
||||
|
||||
echo ""
|
||||
echo "--- Scanning MCP tool handlers for file reads ---"
|
||||
|
||||
MCP_FILE="$ROOT/src/mcp/mcp.c"
|
||||
MCP_READS_OK=true
|
||||
if [ -f "$MCP_FILE" ]; then
|
||||
# Known safe file reads in mcp.c (with line-range context)
|
||||
# - search_code: writes pattern to tmpfile, reads grep output
|
||||
# - get_code_snippet: reads source via read_file_lines (path-contained)
|
||||
# - manage_adr: reads/writes ADR files
|
||||
# - detect_changes: reads git diff output
|
||||
# - update check: reads the version-check HTTP response (fixed buffer)
|
||||
# - HTTP transport: reads the incoming request body (content-length bound)
|
||||
# Count fopen/fread calls and compare against expected
|
||||
FOPEN_COUNT=$(grep -c 'fopen\|fread\|read_file' "$MCP_FILE" 2>/dev/null || echo "0")
|
||||
# Update this when legitimate reads are added. 13 reads audited as of the
|
||||
# search/ADR/Windows-support commits — all path-contained or transport
|
||||
# reads, no new exfiltration surface.
|
||||
EXPECTED_MAX=13
|
||||
if [ "$FOPEN_COUNT" -gt "$EXPECTED_MAX" ]; then
|
||||
echo "REVIEW: src/mcp/mcp.c has $FOPEN_COUNT file read operations (expected max $EXPECTED_MAX)"
|
||||
echo " New file reads in MCP tool handlers must be reviewed for data exfiltration risk."
|
||||
MCP_READS_OK=false
|
||||
fi
|
||||
fi
|
||||
|
||||
if $MCP_READS_OK; then
|
||||
echo "OK: MCP tool handler file reads within expected count."
|
||||
fi
|
||||
|
||||
# ── 6. GitHub Actions pinned to SHA ───────────────────────────────
|
||||
# Mutable tags (@v4) can be moved by compromised maintainers.
|
||||
# All Actions must be pinned to full commit SHAs.
|
||||
|
||||
echo ""
|
||||
echo "--- Scanning for unpinned GitHub Actions ---"
|
||||
|
||||
UNPINNED_FOUND=false
|
||||
if [ -d "$ROOT/.github/workflows" ]; then
|
||||
while IFS= read -r file; do
|
||||
relfile="${file#"$ROOT/"}"
|
||||
# Only check files tracked by git (skip stale untracked files)
|
||||
git ls-files --error-unmatch "$file" > /dev/null 2>&1 || continue
|
||||
while IFS= read -r match; do
|
||||
[[ -z "$match" ]] && continue
|
||||
echo "BLOCKED: ${relfile}: ${match}"
|
||||
echo " -> Pin to SHA: uses: owner/action@<commit-sha> # version"
|
||||
fail
|
||||
UNPINNED_FOUND=true
|
||||
done < <(grep -nE 'uses:.*@v[0-9]' "$file" 2>/dev/null || true)
|
||||
done < <(find "$ROOT/.github/workflows" -name '*.yml' -type f | sort)
|
||||
fi
|
||||
|
||||
if ! $UNPINNED_FOUND; then
|
||||
echo "OK: All GitHub Actions pinned to commit SHAs."
|
||||
fi
|
||||
|
||||
# ── Summary ──────────────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
RESULT=$(cat "$FAIL_FLAG")
|
||||
if [[ "$RESULT" != "0" ]]; then
|
||||
echo "=== SECURITY AUDIT FAILED ==="
|
||||
echo "Fix the issues above or add entries to scripts/security-allowlist.txt with justifications."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Security audit passed ==="
|
||||
Executable
+168
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Fuzz testing: feeds random/mutated inputs to the MCP server and CLI
|
||||
# to find crashes, hangs, and memory errors. Runs for a limited time.
|
||||
#
|
||||
# Usage: scripts/security-fuzz-random.sh <binary-path> [duration_seconds]
|
||||
|
||||
BINARY="${1:?usage: security-fuzz-random.sh <binary-path> [duration_seconds]}"
|
||||
DURATION="${2:-60}"
|
||||
|
||||
if [[ ! -f "$BINARY" ]]; then
|
||||
echo "FAIL: binary not found: $BINARY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Fuzz Testing ($DURATION seconds) ==="
|
||||
|
||||
FUZZ_TMPDIR=$(mktemp -d)
|
||||
trap 'rm -rf "$FUZZ_TMPDIR"' EXIT
|
||||
|
||||
CRASHES=0
|
||||
ITERATIONS=0
|
||||
END_TIME=$((SECONDS + DURATION))
|
||||
|
||||
# Portable timeout
|
||||
run_with_timeout() {
|
||||
local secs="$1"; shift
|
||||
if command -v timeout &>/dev/null; then
|
||||
timeout "$secs" "$@" || true
|
||||
else
|
||||
perl -e "alarm($secs); exec @ARGV" -- "$@" || true
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Phase 1: Random JSON-RPC mutations ───────────────────────
|
||||
echo ""
|
||||
echo "--- Phase 1: Random JSON-RPC mutations ---"
|
||||
|
||||
while [ $SECONDS -lt $END_TIME ]; do
|
||||
ITERATIONS=$((ITERATIONS + 1))
|
||||
|
||||
# Generate a random mutated JSON-RPC payload
|
||||
PAYLOAD=$(python3 -c "
|
||||
import random, string, json
|
||||
|
||||
# Base valid request
|
||||
base = {
|
||||
'jsonrpc': '2.0',
|
||||
'id': random.randint(-999999, 999999),
|
||||
'method': random.choice([
|
||||
'initialize', 'tools/call', 'tools/list',
|
||||
random.choice(string.ascii_letters) * random.randint(1, 100),
|
||||
'',
|
||||
]),
|
||||
}
|
||||
|
||||
# Random mutations
|
||||
mutation = random.randint(0, 10)
|
||||
if mutation == 0:
|
||||
# Valid tool call with random args
|
||||
base['params'] = {'name': 'search_graph', 'arguments': {
|
||||
'name_pattern': ''.join(random.choices(string.printable, k=random.randint(0, 500)))
|
||||
}}
|
||||
elif mutation == 1:
|
||||
# Huge nested object
|
||||
base['params'] = {'a': {'b': {'c': {'d': {'e': 'deep'}}}}}
|
||||
elif mutation == 2:
|
||||
# Random bytes as method
|
||||
base['method'] = ''.join(random.choices(string.printable, k=random.randint(1, 1000)))
|
||||
elif mutation == 3:
|
||||
# Null fields
|
||||
base['method'] = None
|
||||
base['id'] = None
|
||||
elif mutation == 4:
|
||||
# Array instead of object
|
||||
base = [1, 2, 3]
|
||||
elif mutation == 5:
|
||||
# Empty
|
||||
base = {}
|
||||
elif mutation == 6:
|
||||
# Random Cypher query
|
||||
base['params'] = {'name': 'query_graph', 'arguments': {
|
||||
'query': ''.join(random.choices(string.printable, k=random.randint(1, 200)))
|
||||
}}
|
||||
elif mutation == 7:
|
||||
# Very long string values
|
||||
base['params'] = {'name': 'search_graph', 'arguments': {
|
||||
'name_pattern': 'A' * random.randint(10000, 100000)
|
||||
}}
|
||||
elif mutation == 8:
|
||||
# Unicode/binary-like content
|
||||
base['params'] = {'name': 'search_code', 'arguments': {
|
||||
'pattern': ''.join(chr(random.randint(0, 0xFFFF)) for _ in range(100))
|
||||
}}
|
||||
elif mutation == 9:
|
||||
# Random tool name
|
||||
base['params'] = {'name': ''.join(random.choices(string.ascii_letters, k=50)), 'arguments': {}}
|
||||
else:
|
||||
# Completely random JSON
|
||||
base = ''.join(random.choices(string.printable, k=random.randint(1, 500)))
|
||||
|
||||
try:
|
||||
print(json.dumps(base))
|
||||
except:
|
||||
print(json.dumps({'garbage': True}))
|
||||
" 2>/dev/null || echo '{}')
|
||||
|
||||
# Build session: init + mutated payload
|
||||
INIT='{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"fuzz","version":"1.0"}}}'
|
||||
printf '%s\n%s\n' "$INIT" "$PAYLOAD" > "$FUZZ_TMPDIR/input.jsonl"
|
||||
|
||||
# Run and check for crashes (not exit code — we expect errors, just not crashes)
|
||||
run_with_timeout 5 "$BINARY" < "$FUZZ_TMPDIR/input.jsonl" > /dev/null 2>&1
|
||||
EC=$?
|
||||
|
||||
# 139 = SIGSEGV, 134 = SIGABRT, 136 = SIGFPE — real crashes
|
||||
if [ $EC -eq 139 ] || [ $EC -eq 134 ] || [ $EC -eq 136 ]; then
|
||||
echo "CRASH: exit code $EC on iteration $ITERATIONS"
|
||||
echo "Payload: $PAYLOAD"
|
||||
CRASHES=$((CRASHES + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Phase 1: $ITERATIONS iterations, $CRASHES crashes"
|
||||
|
||||
# ── Phase 2: Random Cypher queries via CLI ───────────────────
|
||||
echo ""
|
||||
echo "--- Phase 2: Random Cypher queries via CLI ---"
|
||||
|
||||
CLI_ITERATIONS=0
|
||||
CLI_END=$((SECONDS + 10))
|
||||
|
||||
while [ $SECONDS -lt $CLI_END ]; do
|
||||
CLI_ITERATIONS=$((CLI_ITERATIONS + 1))
|
||||
|
||||
QUERY=$(python3 -c "
|
||||
import random, string
|
||||
parts = ['MATCH', 'WHERE', 'RETURN', '(', ')', '-[', ']->', '<-[', ']-',
|
||||
'n', 'r', '.name', '.label', '=', '\"', \"'\", ';', '--', 'DROP',
|
||||
'DELETE', 'ATTACH', 'DETACH', '*', 'COUNT', 'LIMIT', 'ORDER BY']
|
||||
q = ' '.join(random.choices(parts, k=random.randint(1, 20)))
|
||||
# Sometimes add random garbage
|
||||
if random.random() > 0.5:
|
||||
q += ''.join(random.choices(string.printable, k=random.randint(1, 100)))
|
||||
print(q)
|
||||
" 2>/dev/null || echo "MATCH (n) RETURN n")
|
||||
|
||||
run_with_timeout 3 "$BINARY" cli query_graph "{\"query\":\"$QUERY\"}" > /dev/null 2>&1
|
||||
EC=$?
|
||||
|
||||
if [ $EC -eq 139 ] || [ $EC -eq 134 ] || [ $EC -eq 136 ]; then
|
||||
echo "CRASH: exit code $EC on Cypher iteration $CLI_ITERATIONS"
|
||||
echo "Query: $QUERY"
|
||||
CRASHES=$((CRASHES + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Phase 2: $CLI_ITERATIONS iterations, $CRASHES total crashes"
|
||||
|
||||
# ── Summary ──────────────────────────────────────────────────
|
||||
echo ""
|
||||
if [ $CRASHES -gt 0 ]; then
|
||||
echo "=== FUZZ TESTING FAILED: $CRASHES crash(es) found ==="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Fuzz testing passed: $((ITERATIONS + CLI_ITERATIONS)) iterations, 0 crashes ==="
|
||||
Executable
+130
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Layer 7: MCP robustness test — sends adversarial JSON-RPC payloads via stdio.
|
||||
#
|
||||
# Verifies the MCP server handles malformed, oversized, and crafted inputs
|
||||
# without crashing. Each payload is sent as a complete session (init + payload + EOF).
|
||||
#
|
||||
# Usage: scripts/security-fuzz.sh <binary-path>
|
||||
|
||||
BINARY="${1:?usage: security-fuzz.sh <binary-path>}"
|
||||
|
||||
if [[ ! -f "$BINARY" ]]; then
|
||||
echo "FAIL: binary not found: $BINARY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Layer 7: MCP Robustness Test ==="
|
||||
|
||||
FAIL=0
|
||||
PASS=0
|
||||
TOTAL=0
|
||||
|
||||
# Temp directory for input files (avoids pipe/stdin issues with timeout)
|
||||
FUZZ_TMPDIR=$(mktemp -d)
|
||||
trap 'rm -rf "$FUZZ_TMPDIR"' EXIT
|
||||
|
||||
# Helper: send a payload to the MCP server and check it doesn't crash.
|
||||
# Uses temp file + perl alarm for portable timeout (works on macOS + Linux).
|
||||
test_payload() {
|
||||
local name="$1"
|
||||
local payload="$2"
|
||||
TOTAL=$((TOTAL + 1))
|
||||
|
||||
# Write session input to a temp file (avoids pipe/stdin issues)
|
||||
local tmpinput="$FUZZ_TMPDIR/input_${TOTAL}.jsonl"
|
||||
printf '%s\n%s\n%s\n' \
|
||||
'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"fuzz","version":"1.0"}}}' \
|
||||
'{"jsonrpc":"2.0","method":"notifications/initialized"}' \
|
||||
"$payload" > "$tmpinput"
|
||||
|
||||
# Run with 10s timeout: GNU timeout → perl alarm fallback
|
||||
local ec=0
|
||||
if command -v timeout &>/dev/null; then
|
||||
timeout 10 "$BINARY" < "$tmpinput" > /dev/null 2>&1 || ec=$?
|
||||
else
|
||||
perl -e 'alarm(10); exec @ARGV' -- "$BINARY" < "$tmpinput" > /dev/null 2>&1 || ec=$?
|
||||
fi
|
||||
|
||||
# Acceptable exits:
|
||||
# 0 = clean shutdown on EOF
|
||||
# 141 = SIGPIPE (pipe closed while writing — normal for stdio MCP)
|
||||
# 142 = SIGALRM (perl timeout — hung process, same as GNU timeout 124)
|
||||
# 124 = GNU timeout
|
||||
if [[ $ec -eq 0 || $ec -eq 141 ]]; then
|
||||
PASS=$((PASS + 1))
|
||||
elif [[ $ec -eq 124 || $ec -eq 142 ]]; then
|
||||
echo "FAIL: $name — timed out (hung for 10s)"
|
||||
FAIL=$((FAIL + 1))
|
||||
else
|
||||
echo "FAIL: $name — crashed with exit code $ec"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "--- Malformed JSON ---"
|
||||
|
||||
test_payload "empty line" ""
|
||||
test_payload "garbage" "not json at all"
|
||||
test_payload "truncated json" '{"jsonrpc":"2.0","id":2,"met'
|
||||
test_payload "null byte in json" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_graph","arguments":{"name_pattern":"test\u0000evil"}}}'
|
||||
test_payload "missing method" '{"jsonrpc":"2.0","id":2}'
|
||||
test_payload "missing id" '{"jsonrpc":"2.0","method":"tools/call"}'
|
||||
test_payload "wrong jsonrpc version" '{"jsonrpc":"1.0","id":2,"method":"tools/call","params":{}}'
|
||||
test_payload "array instead of object" '[1,2,3]'
|
||||
test_payload "deeply nested json" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_graph","arguments":{"name_pattern":{"a":{"b":{"c":{"d":{"e":{"f":"deep"}}}}}}}}}'
|
||||
|
||||
echo ""
|
||||
echo "--- Oversized inputs ---"
|
||||
|
||||
# 1MB string argument
|
||||
HUGE=$(python3 -c "print('A' * 1048576)" 2>/dev/null || python3.9 -c "print('A' * 1048576)" 2>/dev/null || echo "AAAA")
|
||||
test_payload "1MB name_pattern" "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/call\",\"params\":{\"name\":\"search_graph\",\"arguments\":{\"name_pattern\":\"$HUGE\"}}}"
|
||||
|
||||
# Very long tool name
|
||||
test_payload "1000-char tool name" "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/call\",\"params\":{\"name\":\"$(python3 -c "print('x' * 1000)" 2>/dev/null || echo 'xxxx')\",\"arguments\":{}}}"
|
||||
|
||||
echo ""
|
||||
echo "--- Tool-specific adversarial inputs ---"
|
||||
|
||||
# search_graph with regex that could cause ReDoS
|
||||
test_payload "ReDoS regex" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_graph","arguments":{"name_pattern":"(a+)+$"}}}'
|
||||
|
||||
# query_graph with SQL injection attempts
|
||||
test_payload "SQL injection in query" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"query_graph","arguments":{"query":"MATCH (n) RETURN n; DROP TABLE nodes; --"}}}'
|
||||
test_payload "ATTACH attempt via query" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"query_graph","arguments":{"query":"ATTACH DATABASE '"'"'/tmp/evil.db'"'"' AS evil"}}}'
|
||||
|
||||
# detect_changes with shell metacharacters in base_branch
|
||||
test_payload "shell injection in base_branch" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"detect_changes","arguments":{"base_branch":"main'\''$(whoami)'\''","project":"nonexistent"}}}'
|
||||
test_payload "shell injection semicolon" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"detect_changes","arguments":{"base_branch":"main; cat /etc/passwd","project":"nonexistent"}}}'
|
||||
test_payload "shell injection pipe" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"detect_changes","arguments":{"base_branch":"main | curl evil.com","project":"nonexistent"}}}'
|
||||
|
||||
# get_code_snippet with path traversal
|
||||
test_payload "path traversal in qualified_name" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"get_code_snippet","arguments":{"qualified_name":"../../../../etc/passwd"}}}'
|
||||
|
||||
# search_code with shell metacharacters in file_pattern
|
||||
test_payload "shell injection in file_pattern" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_code","arguments":{"pattern":"test","file_pattern":"*.py'\'' ; cat /etc/passwd #"}}}'
|
||||
|
||||
# index_repository with non-existent path (should return error, not crash)
|
||||
test_payload "index nonexistent path" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"index_repository","arguments":{"repo_path":"/nonexistent/path/abc123"}}}'
|
||||
|
||||
# Negative/zero values for numeric params
|
||||
test_payload "negative limit" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_graph","arguments":{"name_pattern":"test","limit":-1}}}'
|
||||
test_payload "zero max_depth" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"trace_path","arguments":{"function_name":"test","max_depth":0}}}'
|
||||
test_payload "huge max_rows" '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"query_graph","arguments":{"query":"MATCH (n) RETURN n","max_rows":999999999}}}'
|
||||
|
||||
echo ""
|
||||
echo "--- Results ---"
|
||||
echo " $PASS/$TOTAL passed"
|
||||
|
||||
if [[ $FAIL -gt 0 ]]; then
|
||||
echo " $FAIL FAILED"
|
||||
echo ""
|
||||
echo "=== MCP ROBUSTNESS TEST FAILED ==="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== MCP robustness test passed ==="
|
||||
Executable
+169
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Layer 4: Install output audit — verifies install --dry-run writes only to expected paths.
|
||||
#
|
||||
# Checks:
|
||||
# 1. All output file paths are in the expected set
|
||||
# 2. No writes to sensitive directories (~/.ssh, ~/.gnupg, ~/.aws, /etc, /usr)
|
||||
# 3. Skill file content contains no dangerous patterns
|
||||
#
|
||||
# Usage: scripts/security-install.sh <binary-path>
|
||||
|
||||
BINARY="${1:?usage: security-install.sh <binary-path>}"
|
||||
|
||||
if [[ ! -f "$BINARY" ]]; then
|
||||
echo "FAIL: binary not found: $BINARY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Layer 4: Install Output Audit ==="
|
||||
|
||||
TMPDIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
|
||||
# Set HOME to tmpdir so install writes there instead of real home
|
||||
export HOME="$TMPDIR/home"
|
||||
mkdir -p "$HOME"
|
||||
|
||||
FAIL=0
|
||||
|
||||
# ── 1. Run install and capture written files ─────────────────────
|
||||
|
||||
echo "--- Running install -y ---"
|
||||
|
||||
# Run install (non-interactive with -y flag)
|
||||
"$BINARY" install -y > "$TMPDIR/install_output.txt" 2>&1 || true
|
||||
|
||||
echo "Install output:"
|
||||
cat "$TMPDIR/install_output.txt"
|
||||
echo ""
|
||||
|
||||
# ── 2. Verify written paths are in expected set ──────────────────
|
||||
|
||||
echo "--- Verifying written file paths ---"
|
||||
|
||||
# Find all files created under HOME
|
||||
find "$HOME" -type f > "$TMPDIR/created_files.txt" 2>/dev/null || true
|
||||
|
||||
# Expected path patterns (relative to HOME):
|
||||
# .config/*/mcp.json (or .mcp.json variants)
|
||||
# .claude/skills/*
|
||||
# .claude/settings.json
|
||||
# .continue/config.yaml
|
||||
# .codeium/config.json
|
||||
# .local/bin/codebase-memory-mcp
|
||||
# Various agent config dirs
|
||||
|
||||
EXPECTED_PATTERNS=(
|
||||
".claude/"
|
||||
".cursor/"
|
||||
".config/"
|
||||
".continue/"
|
||||
".codeium/"
|
||||
".windsurf/"
|
||||
".trae/"
|
||||
".aider/"
|
||||
".local/bin/"
|
||||
"AGENTS.md"
|
||||
"CONVENTIONS.md"
|
||||
".mcp.json"
|
||||
"mcp.json"
|
||||
".zshrc"
|
||||
".bashrc"
|
||||
".profile"
|
||||
)
|
||||
|
||||
while IFS= read -r filepath; do
|
||||
relpath="${filepath#"$HOME/"}"
|
||||
|
||||
matched=false
|
||||
for pattern in "${EXPECTED_PATTERNS[@]}"; do
|
||||
if [[ "$relpath" == *"$pattern"* ]]; then
|
||||
matched=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if ! $matched; then
|
||||
echo "REVIEW: Unexpected file created: $relpath"
|
||||
fi
|
||||
done < "$TMPDIR/created_files.txt"
|
||||
|
||||
# ── 3. Check for writes to sensitive paths ───────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- Checking for sensitive path writes ---"
|
||||
|
||||
SENSITIVE_DIRS=(".ssh" ".gnupg" ".aws" ".kube" ".config/gcloud")
|
||||
|
||||
for dir in "${SENSITIVE_DIRS[@]}"; do
|
||||
if [[ -d "$HOME/$dir" ]]; then
|
||||
echo "BLOCKED: Install created sensitive directory: ~/$dir"
|
||||
FAIL=1
|
||||
fi
|
||||
done
|
||||
|
||||
# Also check install output for any references to sensitive paths
|
||||
for dir in "${SENSITIVE_DIRS[@]}"; do
|
||||
if grep -q "$dir" "$TMPDIR/install_output.txt" 2>/dev/null; then
|
||||
echo "BLOCKED: Install output references sensitive path: $dir"
|
||||
FAIL=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $FAIL -eq 0 ]]; then
|
||||
echo "OK: No sensitive path writes detected."
|
||||
fi
|
||||
|
||||
# ── 4. Audit skill file content ──────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- Auditing skill file content ---"
|
||||
|
||||
SKILLS_DIR="${CLAUDE_CONFIG_DIR:-$HOME/.claude}/skills"
|
||||
if [[ -d "$SKILLS_DIR" ]]; then
|
||||
SKILL_ISSUES=0
|
||||
|
||||
while IFS= read -r skill_file; do
|
||||
basename=$(basename "$skill_file")
|
||||
|
||||
# Check for dangerous patterns in skill content
|
||||
for pattern in 'system(' 'eval(' 'exec(' '__import__(' 'subprocess' 'os.popen'; do
|
||||
if grep -q "$pattern" "$skill_file" 2>/dev/null; then
|
||||
echo "BLOCKED: Skill '$basename' contains dangerous pattern: $pattern"
|
||||
SKILL_ISSUES=1
|
||||
FAIL=1
|
||||
fi
|
||||
done
|
||||
|
||||
# Check for unexpected URLs
|
||||
if grep -oE 'https?://[^\s"'"'"']+' "$skill_file" 2>/dev/null | grep -v 'github.com/DeusData' | grep -v 'localhost' | grep -v '127.0.0.1' > /tmp/sec_skill_urls 2>/dev/null; then
|
||||
while IFS= read -r url; do
|
||||
echo "REVIEW: Skill '$basename' contains URL: $url"
|
||||
done < /tmp/sec_skill_urls
|
||||
rm -f /tmp/sec_skill_urls
|
||||
fi
|
||||
|
||||
# Check for encoded strings (base64-like blocks > 50 chars)
|
||||
if grep -E '[A-Za-z0-9+/]{50,}={0,2}' "$skill_file" > /dev/null 2>&1; then
|
||||
echo "REVIEW: Skill '$basename' contains potential encoded content"
|
||||
fi
|
||||
done < <(find "$SKILLS_DIR" -type f -name '*.md')
|
||||
|
||||
if [[ $SKILL_ISSUES -eq 0 ]]; then
|
||||
echo "OK: Skill files contain no dangerous patterns."
|
||||
fi
|
||||
else
|
||||
echo "SKIP: No skills directory created."
|
||||
fi
|
||||
|
||||
# ── Summary ──────────────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
if [[ $FAIL -ne 0 ]]; then
|
||||
echo "=== INSTALL OUTPUT AUDIT FAILED ==="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Install output audit passed ==="
|
||||
Executable
+107
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Layer 3: Network egress test — monitors outbound connections during MCP session.
|
||||
#
|
||||
# Runs a full MCP session (initialize → index → search → EOF) under strace
|
||||
# and verifies only expected connections are made.
|
||||
#
|
||||
# Linux only (strace required). macOS/Windows: skip with success.
|
||||
#
|
||||
# Usage: scripts/security-network.sh <binary-path>
|
||||
|
||||
BINARY="${1:?usage: security-network.sh <binary-path>}"
|
||||
|
||||
if [[ ! -f "$BINARY" ]]; then
|
||||
echo "FAIL: binary not found: $BINARY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Layer 3: Network Egress Test ==="
|
||||
|
||||
# Skip on non-Linux (no strace)
|
||||
if [[ "$(uname)" != "Linux" ]]; then
|
||||
echo "SKIP: strace not available on $(uname) — covered by binary string audit"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! command -v strace &>/dev/null; then
|
||||
echo "SKIP: strace not installed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TMPDIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
|
||||
# Create a minimal test project
|
||||
mkdir -p "$TMPDIR/project/src"
|
||||
cat > "$TMPDIR/project/src/main.py" << 'EOF'
|
||||
def main():
|
||||
print("hello")
|
||||
EOF
|
||||
|
||||
# MCP session input (jsonrpc over stdio)
|
||||
cat > "$TMPDIR/input.jsonl" << JSONL
|
||||
{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"test","version":"1.0"}}}
|
||||
{"jsonrpc":"2.0","method":"notifications/initialized"}
|
||||
{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"index_repository","arguments":{"repo_path":"$TMPDIR/project"}}}
|
||||
{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"search_graph","arguments":{"name_pattern":"main"}}}
|
||||
JSONL
|
||||
|
||||
STRACE_LOG="$TMPDIR/strace.log"
|
||||
|
||||
echo "Running MCP session under strace..."
|
||||
|
||||
# Run binary with strace monitoring connect() syscalls
|
||||
# -f: follow forks, -e trace=connect: only log connect() calls
|
||||
timeout 30 strace -f -e trace=connect \
|
||||
"$BINARY" < "$TMPDIR/input.jsonl" \
|
||||
> "$TMPDIR/output.jsonl" 2> "$STRACE_LOG" || true
|
||||
|
||||
echo ""
|
||||
echo "--- Connection log ---"
|
||||
|
||||
FAIL=0
|
||||
|
||||
# Parse strace output for connect() calls with AF_INET (IPv4 network)
|
||||
# Format: connect(fd, {sa_family=AF_INET, sin_port=htons(443), sin_addr=inet_addr("x.x.x.x")}, ...)
|
||||
if grep 'sa_family=AF_INET' "$STRACE_LOG" > "$TMPDIR/connections.log" 2>/dev/null; then
|
||||
while IFS= read -r conn; do
|
||||
# Extract destination IP and port
|
||||
ip=$(echo "$conn" | grep -oP 'inet_addr\("\K[^"]+' || echo "unknown")
|
||||
port=$(echo "$conn" | grep -oP 'htons\(\K[0-9]+' || echo "0")
|
||||
|
||||
# Allowed connections:
|
||||
# - 127.0.0.1 (localhost, any port)
|
||||
# - DNS (port 53, any IP)
|
||||
# - api.github.com (140.82.x.x range, port 443) — update check
|
||||
case "$ip" in
|
||||
127.0.0.1|0.0.0.0)
|
||||
echo " OK: localhost:$port"
|
||||
;;
|
||||
*)
|
||||
if [[ "$port" == "53" ]]; then
|
||||
echo " OK: DNS lookup to $ip"
|
||||
elif [[ "$port" == "443" ]]; then
|
||||
echo " REVIEW: HTTPS to $ip:$port (expected: api.github.com for update check)"
|
||||
# This is expected — the binary checks for updates on startup
|
||||
else
|
||||
echo " BLOCKED: Unexpected connection to $ip:$port"
|
||||
FAIL=1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done < "$TMPDIR/connections.log"
|
||||
else
|
||||
echo " No outbound connections detected."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [[ $FAIL -ne 0 ]]; then
|
||||
echo "=== NETWORK EGRESS TEST FAILED ==="
|
||||
echo "Unexpected outbound connections detected. Full strace log:"
|
||||
grep 'connect(' "$STRACE_LOG" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Network egress test passed ==="
|
||||
Executable
+198
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Layer 2: Binary string audit — post-build check on the production binary.
|
||||
#
|
||||
# Scans extracted strings for:
|
||||
# 1. Unauthorized URLs (only github.com + localhost allowed)
|
||||
# 2. Suspiciously long base64-encoded payloads
|
||||
# 3. Dangerous command names (wget, nc, netcat, telnet, ssh, /dev/tcp)
|
||||
# 4. Credential patterns (password=, secret=, api_key=)
|
||||
#
|
||||
# Usage: scripts/security-strings.sh <binary-path>
|
||||
|
||||
BINARY="${1:?usage: security-strings.sh <binary-path>}"
|
||||
|
||||
if [[ ! -f "$BINARY" ]]; then
|
||||
echo "FAIL: binary not found: $BINARY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FAIL=0
|
||||
|
||||
# Detect file type. Shell scripts and other text files extract poorly via
|
||||
# `strings` (the entire content becomes the "strings"), and rules tuned for
|
||||
# compiled binaries (URL allowlist, wget/telnet detection) produce false
|
||||
# positives — install.sh legitimately uses `wget` as a curl fallback, and
|
||||
# `case` glob patterns like `https://*` look like unauthorized URLs.
|
||||
#
|
||||
# Strategy: for non-binary files we still run credential and base64 audits
|
||||
# (those are universally meaningful), but skip the URL and dangerous-command
|
||||
# audits which are designed for compiled artifacts. Script content is
|
||||
# reviewed in PRs and scanned end-to-end by VirusTotal in the same pipeline.
|
||||
IS_SCRIPT=false
|
||||
if command -v file &>/dev/null; then
|
||||
FILE_TYPE=$(file -b "$BINARY" 2>/dev/null || echo "")
|
||||
case "$FILE_TYPE" in
|
||||
*"shell script"*|*"ASCII text"*|*"UTF-8 Unicode text"*|*"Unicode text"*|*"a /usr/bin/env"*)
|
||||
IS_SCRIPT=true
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
if $IS_SCRIPT; then
|
||||
echo "=== Layer 2: Script Content Audit ==="
|
||||
else
|
||||
echo "=== Layer 2: Binary String Audit ==="
|
||||
fi
|
||||
echo "File: $BINARY"
|
||||
echo ""
|
||||
|
||||
# Check for strings command (needs binutils on some MSYS2 setups)
|
||||
if ! command -v strings &>/dev/null; then
|
||||
echo "SKIP: 'strings' command not available"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Extract all printable strings (min length 4)
|
||||
STRINGS_FILE=$(mktemp)
|
||||
SEC_CMDS=$(mktemp)
|
||||
SEC_CREDS=$(mktemp)
|
||||
trap 'rm -f "$STRINGS_FILE" "$SEC_CMDS" "$SEC_CREDS"' EXIT
|
||||
strings -n 4 "$BINARY" | sort -u > "$STRINGS_FILE"
|
||||
|
||||
# ── 1. URL audit (binary only — scripts handled via VT + PR review) ────
|
||||
|
||||
if $IS_SCRIPT; then
|
||||
echo "--- URL audit (skipped — script file) ---"
|
||||
else
|
||||
echo "--- URL audit ---"
|
||||
|
||||
# Allowed URL prefixes
|
||||
ALLOWED_URLS=(
|
||||
"https://api.github.com/repos/DeusData/codebase-memory-mcp"
|
||||
"https://github.com/DeusData/codebase-memory-mcp"
|
||||
"http://127.0.0.1"
|
||||
"http://localhost"
|
||||
# SQLite internal URLs (part of vendored sqlite3 strings)
|
||||
"https://sqlite.org"
|
||||
"https://www.sqlite.org"
|
||||
# Toolchain URLs embedded by compiler/linker in static builds
|
||||
"https://bugs.launchpad.net"
|
||||
"https://gcc.gnu.org"
|
||||
"https://sourceware.org"
|
||||
# MSYS2 CLANG64 toolchain (libc++/compiler-rt) package-tracker URL, baked
|
||||
# into the static Windows .exe — Windows-only, hence Linux smoke stays clean.
|
||||
"https://github.com/msys2/MINGW-packages"
|
||||
# W3C XML namespace URIs (SVG, MathML, XLink — used in UI bundle)
|
||||
"http://www.w3.org/"
|
||||
# UI bundle: React, Three.js, Tailwind, Google Fonts, bundled libraries
|
||||
"https://react.dev"
|
||||
"https://fonts.googleapis.com"
|
||||
"https://fonts.gstatic.com"
|
||||
"https://tailwindcss.com"
|
||||
"https://cdn.jsdelivr.net"
|
||||
"https://docs.pmnd.rs"
|
||||
"https://jcgt.org"
|
||||
"https://github.com/pmndrs"
|
||||
"https://github.com/react-spring"
|
||||
"https://github.com/101arrowz"
|
||||
"https://github.com/arty-name"
|
||||
"https://github.com/fredli74"
|
||||
"https://github.com/lojjic"
|
||||
)
|
||||
|
||||
while IFS= read -r url; do
|
||||
# Skip short false positives from binary data (e.g. "https://H9")
|
||||
if [[ ${#url} -lt 15 ]]; then
|
||||
continue
|
||||
fi
|
||||
allowed=false
|
||||
for prefix in "${ALLOWED_URLS[@]}"; do
|
||||
if [[ "$url" == "$prefix"* ]]; then
|
||||
allowed=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if ! $allowed; then
|
||||
echo "BLOCKED: Unauthorized URL in binary: $url"
|
||||
FAIL=1
|
||||
fi
|
||||
done < <(grep -oE 'https?://[a-zA-Z0-9._/~:@!$&()*+,;=?#%[-]+' "$STRINGS_FILE" || true)
|
||||
|
||||
if [[ $FAIL -eq 0 ]]; then
|
||||
echo "OK: All URLs are authorized."
|
||||
fi
|
||||
fi # end !IS_SCRIPT URL audit
|
||||
|
||||
# ── 2. Base64 payload detection ──────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- Base64 payload detection ---"
|
||||
|
||||
# Look for base64-like strings longer than 100 chars (potential encoded payloads)
|
||||
B64_COUNT=$(grep -cE '^[A-Za-z0-9+/]{100,}={0,2}$' "$STRINGS_FILE" || true)
|
||||
if [[ "$B64_COUNT" -gt 0 ]]; then
|
||||
echo "WARNING: Found $B64_COUNT potential base64-encoded strings > 100 chars"
|
||||
grep -E '^[A-Za-z0-9+/]{100,}={0,2}$' "$STRINGS_FILE" | head -5 | while IFS= read -r s; do
|
||||
echo " ${s:0:80}..."
|
||||
done
|
||||
# Warning only — tree-sitter grammar data can look like base64
|
||||
else
|
||||
echo "OK: No suspicious base64 payloads found."
|
||||
fi
|
||||
|
||||
# ── 3. Dangerous command detection (binary only) ─────────────────
|
||||
|
||||
echo ""
|
||||
if $IS_SCRIPT; then
|
||||
echo "--- Dangerous command detection (skipped — script file) ---"
|
||||
DANGEROUS_CMDS=''
|
||||
else
|
||||
echo "--- Dangerous command detection ---"
|
||||
|
||||
DANGEROUS_CMDS='wget|netcat|ncat|/dev/tcp|telnet'
|
||||
# Known-benign matches (vendored grammar URI scheme tables, etc.). Each entry
|
||||
# is a regex matched against the full line; matches are stripped before
|
||||
# evaluation. Document the source so reviewers can verify the false positive.
|
||||
ALLOWED_DANGEROUS=(
|
||||
'^telnet$' # rst tree-sitter grammar: valid_schemas[] in vendored/grammars/rst/tree_sitter_rst/chars.c
|
||||
)
|
||||
if grep -wE "$DANGEROUS_CMDS" "$STRINGS_FILE" > "$SEC_CMDS" 2>/dev/null && [ -s "$SEC_CMDS" ]; then
|
||||
for allow in "${ALLOWED_DANGEROUS[@]}"; do
|
||||
grep -vE "$allow" "$SEC_CMDS" > "${SEC_CMDS}.tmp" || true
|
||||
mv "${SEC_CMDS}.tmp" "$SEC_CMDS"
|
||||
done
|
||||
fi
|
||||
if [ -s "$SEC_CMDS" ]; then
|
||||
echo "BLOCKED: Dangerous commands found in binary:"
|
||||
cat "$SEC_CMDS"
|
||||
FAIL=1
|
||||
else
|
||||
echo "OK: No dangerous commands found."
|
||||
fi
|
||||
fi # end !IS_SCRIPT dangerous-cmd audit
|
||||
|
||||
# ── 4. Credential pattern detection ──────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- Credential pattern detection ---"
|
||||
|
||||
CRED_PATTERNS='password=|secret=|api_key=|apikey=|auth_token=|private_key='
|
||||
if grep -iE "$CRED_PATTERNS" "$STRINGS_FILE" > "$SEC_CREDS" 2>/dev/null; then
|
||||
echo "BLOCKED: Credential patterns found in binary:"
|
||||
cat "$SEC_CREDS"
|
||||
FAIL=1
|
||||
else
|
||||
echo "OK: No credential patterns found."
|
||||
fi
|
||||
|
||||
# ── Summary ──────────────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
if [[ $FAIL -ne 0 ]]; then
|
||||
echo "=== BINARY STRING AUDIT FAILED ==="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Binary string audit passed ==="
|
||||
Executable
+211
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Layer 6: Graph UI security audit.
|
||||
#
|
||||
# Audits:
|
||||
# A. Frontend asset scan (embedded JS/CSS/HTML or graph-ui/dist/)
|
||||
# B. HTTP server binding (must be 127.0.0.1 only)
|
||||
# C. RPC proxy scope (no system()/popen() in HTTP handler path)
|
||||
# D. CORS check (no wildcard Access-Control-Allow-Origin)
|
||||
#
|
||||
# Usage: scripts/security-ui.sh
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
FAIL=0
|
||||
|
||||
# Use mktemp for all temp files (cross-platform safe)
|
||||
SEC_TMPDIR=$(mktemp -d)
|
||||
trap 'rm -rf "$SEC_TMPDIR"' EXIT
|
||||
|
||||
echo "=== Layer 6: Graph UI Security Audit ==="
|
||||
|
||||
# ── A. Frontend asset scan ───────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- A. Frontend asset scan ---"
|
||||
|
||||
# Check both built dist and embedded asset source
|
||||
UI_DIRS=()
|
||||
[[ -d "$ROOT/graph-ui/dist" ]] && UI_DIRS+=("$ROOT/graph-ui/dist")
|
||||
[[ -d "$ROOT/graph-ui/src" ]] && UI_DIRS+=("$ROOT/graph-ui/src")
|
||||
|
||||
if [[ ${#UI_DIRS[@]} -eq 0 ]]; then
|
||||
echo "SKIP: No graph-ui directory found."
|
||||
else
|
||||
for UI_DIR in "${UI_DIRS[@]}"; do
|
||||
echo "Scanning: $UI_DIR"
|
||||
|
||||
# A1: No external domains in JS/CSS/TS source code.
|
||||
# For src/ (our code): strict — any external URL is blocked.
|
||||
# For dist/ (bundled npm output): skip inline URL scan — minified JS
|
||||
# contains hundreds of string-constant URLs from libraries (React error
|
||||
# pages, W3C namespace URIs, CDN references, OSS credits) that are never
|
||||
# fetched at runtime. Scanning them is noise. Structural checks (A2-A6)
|
||||
# still apply to dist/.
|
||||
is_dist=false
|
||||
[[ "$UI_DIR" == *"/dist" || "$UI_DIR" == *"/dist/" ]] && is_dist=true
|
||||
|
||||
if ! $is_dist; then
|
||||
echo " Checking for external domains (source)..."
|
||||
if find "$UI_DIR" -type f \( -name '*.js' -o -name '*.ts' -o -name '*.tsx' -o -name '*.css' \) -exec grep -lE 'https?://' {} \; 2>/dev/null | head -20 > "$SEC_TMPDIR/urls"; then
|
||||
while IFS= read -r file; do
|
||||
relfile="${file#"$ROOT/"}"
|
||||
grep -onE 'https?://[^\s"'"'"')]+' "$file" 2>/dev/null | while IFS=: read -r lineno url; do
|
||||
case "$url" in
|
||||
http://localhost*|http://127.0.0.1*|https://localhost*|https://127.0.0.1*)
|
||||
;; # OK — local dev/runtime
|
||||
*)
|
||||
echo " BLOCKED: ${relfile}:${lineno}: External URL: $url"
|
||||
touch "$SEC_TMPDIR/fail_flag"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
done < "$SEC_TMPDIR/urls"
|
||||
fi
|
||||
[[ -f "$SEC_TMPDIR/fail_flag" ]] && FAIL=1 && rm -f "$SEC_TMPDIR/fail_flag"
|
||||
else
|
||||
echo " Skipping inline URL scan for dist/ (bundled library strings)."
|
||||
echo " Structural checks (script loads, tracking, eval, iframes) still apply."
|
||||
fi
|
||||
|
||||
# A2: No external script/link loads in HTML
|
||||
echo " Checking for external script/link loads..."
|
||||
if find "$UI_DIR" -type f -name '*.html' -exec grep -lE '<script\s+src=|<link\s+href=' {} \; 2>/dev/null > "$SEC_TMPDIR/scripts"; then
|
||||
while IFS= read -r file; do
|
||||
relfile="${file#"$ROOT/"}"
|
||||
if grep -nE '<script\s+src="https?://|<link\s+href="https?://' "$file" 2>/dev/null \
|
||||
| grep -v 'localhost' | grep -v '127.0.0.1' \
|
||||
| grep -v 'fonts.googleapis.com' | grep -v 'fonts.gstatic.com'; then
|
||||
echo " BLOCKED: ${relfile}: External script/link load detected"
|
||||
FAIL=1
|
||||
fi
|
||||
done < "$SEC_TMPDIR/scripts"
|
||||
fi
|
||||
|
||||
# A3: No tracking/analytics
|
||||
echo " Checking for tracking/analytics..."
|
||||
TRACKING='google-analytics|gtag|mixpanel|segment\.com|hotjar|sentry\.io|plausible|posthog'
|
||||
if find "$UI_DIR" -type f \( -name '*.js' -o -name '*.ts' -o -name '*.tsx' -o -name '*.html' \) \
|
||||
-exec grep -lE "$TRACKING" {} \; 2>/dev/null > "$SEC_TMPDIR/track"; then
|
||||
while IFS= read -r file; do
|
||||
relfile="${file#"$ROOT/"}"
|
||||
echo " BLOCKED: ${relfile}: Tracking/analytics reference found"
|
||||
grep -nE "$TRACKING" "$file" | head -3
|
||||
FAIL=1
|
||||
done < "$SEC_TMPDIR/track"
|
||||
fi
|
||||
|
||||
# A4: No hidden iframes
|
||||
echo " Checking for iframes..."
|
||||
if find "$UI_DIR" -type f -name '*.html' -exec grep -li '<iframe' {} \; 2>/dev/null > "$SEC_TMPDIR/iframe"; then
|
||||
while IFS= read -r file; do
|
||||
relfile="${file#"$ROOT/"}"
|
||||
echo " BLOCKED: ${relfile}: iframe detected"
|
||||
FAIL=1
|
||||
done < "$SEC_TMPDIR/iframe"
|
||||
fi
|
||||
|
||||
# A5: No eval/Function constructor in JS
|
||||
echo " Checking for eval/Function constructor..."
|
||||
if find "$UI_DIR" -type f \( -name '*.js' -o -name '*.ts' -o -name '*.tsx' \) \
|
||||
-exec grep -nE '\beval\s*\(|new\s+Function\s*\(' {} \; 2>/dev/null | grep -v node_modules | grep -v '\.test\.' > "$SEC_TMPDIR/eval"; then
|
||||
while IFS= read -r match; do
|
||||
echo " REVIEW: eval/Function found: $match"
|
||||
done < "$SEC_TMPDIR/eval"
|
||||
fi
|
||||
|
||||
# A6: No WebSocket to external
|
||||
echo " Checking for external WebSocket connections..."
|
||||
if find "$UI_DIR" -type f \( -name '*.js' -o -name '*.ts' -o -name '*.tsx' \) \
|
||||
-exec grep -nE 'wss?://' {} \; 2>/dev/null | grep -v 'localhost' | grep -v '127.0.0.1' > "$SEC_TMPDIR/ws"; then
|
||||
while IFS= read -r match; do
|
||||
echo " BLOCKED: External WebSocket: $match"
|
||||
FAIL=1
|
||||
done < "$SEC_TMPDIR/ws"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ── B. HTTP server binding check ─────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- B. HTTP server binding check ---"
|
||||
|
||||
# The UI server is split across two first-party files:
|
||||
# httpd.c — socket transport (binding, parsing)
|
||||
# http_server.c — routing + endpoint handlers (CORS policy)
|
||||
# Both MUST exist — a missing file means the audit target moved and the
|
||||
# audit would go blind, so that is a hard failure, not a skip.
|
||||
HTTPD="$ROOT/src/ui/httpd.c"
|
||||
HTTP_SERVER="$ROOT/src/ui/http_server.c"
|
||||
for f in "$HTTPD" "$HTTP_SERVER"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "BLOCKED: expected UI server source missing: $f"
|
||||
FAIL=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -f "$HTTPD" ]]; then
|
||||
# Must bind to 127.0.0.1 only
|
||||
if grep -q '127\.0\.0\.1' "$HTTPD"; then
|
||||
echo "OK: Server binds to 127.0.0.1"
|
||||
else
|
||||
echo "BLOCKED: No 127.0.0.1 binding found in httpd.c"
|
||||
FAIL=1
|
||||
fi
|
||||
|
||||
# Must NOT bind to 0.0.0.0 or INADDR_ANY
|
||||
if cat "$HTTPD" "$HTTP_SERVER" 2>/dev/null | grep -E '0\.0\.0\.0|INADDR_ANY|in6addr_any' | grep -v '^\s*//' | grep -v '^\s*\*' > /dev/null 2>&1; then
|
||||
echo "BLOCKED: Server may bind to all interfaces (0.0.0.0/INADDR_ANY found)"
|
||||
FAIL=1
|
||||
else
|
||||
echo "OK: No 0.0.0.0/INADDR_ANY binding"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── C. RPC proxy scope check ─────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- C. RPC proxy scope check ---"
|
||||
|
||||
if [[ -f "$HTTP_SERVER" ]]; then
|
||||
# The HTTP handler should not directly call system()/popen()
|
||||
# (fork/execl for indexing is allowed as it's tracked)
|
||||
if cat "$HTTPD" "$HTTP_SERVER" 2>/dev/null | grep -n 'system(' | grep -v '^\s*//' | grep -v '^\s*\*' > /dev/null 2>&1; then
|
||||
echo "BLOCKED: system() call found in HTTP server (use subprocess instead)"
|
||||
FAIL=1
|
||||
else
|
||||
echo "OK: No system() calls in HTTP handler"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── D. CORS check ────────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- D. CORS check ---"
|
||||
|
||||
if [[ -f "$HTTP_SERVER" ]]; then
|
||||
if cat "$HTTPD" "$HTTP_SERVER" 2>/dev/null | grep -E 'Allow-Origin:\s*\*' | grep -v '^\s*//' | grep -v '^\s*\*' > /dev/null 2>&1; then
|
||||
echo "BLOCKED: CORS wildcard (Access-Control-Allow-Origin: *) found"
|
||||
echo " This allows any website to access the local server."
|
||||
FAIL=1
|
||||
else
|
||||
echo "OK: No CORS wildcard found"
|
||||
fi
|
||||
|
||||
# Check that CORS reflects localhost origins
|
||||
if grep -q 'localhost' "$HTTP_SERVER" && grep -q 'update_cors\|Access-Control-Allow-Origin' "$HTTP_SERVER"; then
|
||||
echo "OK: CORS appears to validate localhost origins"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Summary ──────────────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
if [[ $FAIL -ne 0 ]]; then
|
||||
echo "=== UI SECURITY AUDIT FAILED ==="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== UI security audit passed ==="
|
||||
Executable
+202
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Layer 8: Vendored dependency integrity — verifies vendored C sources match
|
||||
# checked-in checksums. Detects supply chain tampering of vendored libraries.
|
||||
#
|
||||
# Libraries covered: mimalloc, sqlite3, tre, xxhash, yyjson
|
||||
#
|
||||
# Usage: scripts/security-vendored.sh
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
CHECKSUMS="$ROOT/scripts/vendored-checksums.txt"
|
||||
|
||||
if [[ ! -f "$CHECKSUMS" ]]; then
|
||||
echo "FAIL: checksums file not found: $CHECKSUMS"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Layer 8: Vendored Dependency Integrity ==="
|
||||
|
||||
# Detect shasum command (shasum on macOS, sha256sum on Linux)
|
||||
if command -v sha256sum &>/dev/null; then
|
||||
SHA_CMD="sha256sum"
|
||||
elif command -v shasum &>/dev/null; then
|
||||
SHA_CMD="shasum -a 256"
|
||||
else
|
||||
echo "SKIP: no sha256sum or shasum available"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
FAIL=0
|
||||
CHECKED=0
|
||||
MISSING=0
|
||||
|
||||
# Verify each file in the checksums list
|
||||
while IFS=' ' read -r expected_hash filepath; do
|
||||
# Skip empty lines
|
||||
[[ -z "$expected_hash" ]] && continue
|
||||
|
||||
# Strip the two-space separator from filepath (sha256sum format: "hash file")
|
||||
filepath="${filepath#"${filepath%%[![:space:]]*}"}"
|
||||
|
||||
full_path="$ROOT/$filepath"
|
||||
if [[ ! -f "$full_path" ]]; then
|
||||
echo "MISSING: $filepath"
|
||||
MISSING=$((MISSING + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
actual_hash=$($SHA_CMD "$full_path" | cut -d' ' -f1)
|
||||
CHECKED=$((CHECKED + 1))
|
||||
|
||||
if [[ "$actual_hash" != "$expected_hash" ]]; then
|
||||
echo "MISMATCH: $filepath"
|
||||
echo " expected: $expected_hash"
|
||||
echo " actual: $actual_hash"
|
||||
FAIL=1
|
||||
fi
|
||||
done < "$CHECKSUMS"
|
||||
|
||||
# Verify every vendored library directory has checksum coverage.
|
||||
# If someone adds a new vendored library, this forces them to register it.
|
||||
echo ""
|
||||
echo "--- Checking vendored library coverage ---"
|
||||
while IFS= read -r libdir; do
|
||||
libname=$(basename "$libdir")
|
||||
# Check if any file from this library is in the checksums
|
||||
if ! grep -q "vendored/${libname}/" "$CHECKSUMS" 2>/dev/null; then
|
||||
echo "BLOCKED: vendored/${libname}/ has NO checksum coverage"
|
||||
echo " Run: scripts/security-vendored.sh --update"
|
||||
FAIL=1
|
||||
fi
|
||||
done < <(find "$ROOT/vendored" -mindepth 1 -maxdepth 1 -type d | sort)
|
||||
|
||||
# Also check for unexpected NEW files in vendored/ that aren't in the checksums
|
||||
UNEXPECTED=0
|
||||
while IFS= read -r file; do
|
||||
relpath="${file#"$ROOT/"}"
|
||||
if ! grep -q "$relpath" "$CHECKSUMS" 2>/dev/null; then
|
||||
echo "NEW FILE: $relpath (not in checksums — run 'scripts/security-vendored.sh --update' to add)"
|
||||
UNEXPECTED=$((UNEXPECTED + 1))
|
||||
fi
|
||||
done < <(find "$ROOT/vendored" -type f \( -name '*.c' -o -name '*.h' \) | sort)
|
||||
|
||||
echo ""
|
||||
echo "Checked: $CHECKED files"
|
||||
[[ $MISSING -gt 0 ]] && echo "Missing: $MISSING files"
|
||||
[[ $UNEXPECTED -gt 0 ]] && echo "New (untracked): $UNEXPECTED files"
|
||||
|
||||
# Handle --update flag: regenerate checksums
|
||||
# ── Dangerous call scan: vendored code must not contain subprocess calls ───
|
||||
|
||||
echo ""
|
||||
echo "--- Scanning vendored code for dangerous calls ---"
|
||||
|
||||
# Subprocess spawning: must not exist in ANY vendored library
|
||||
SUBPROCESS_FUNCS='[^a-z_]system\(|[^a-z]popen\(|[^a-z_]execl\(|[^a-z_]execv\(|[^a-z_]fork\('
|
||||
if grep -rn -E "$SUBPROCESS_FUNCS" "$ROOT/vendored/" --include='*.c' --include='*.h' 2>/dev/null \
|
||||
| grep -v '^\s*//' | grep -v '^\s*\*' | grep -v '#define' | grep -v 'typedef' \
|
||||
| grep -v 'indicating that a fork' > /dev/null 2>&1; then
|
||||
echo "BLOCKED: Subprocess calls found in vendored code:"
|
||||
grep -rn -E "$SUBPROCESS_FUNCS" "$ROOT/vendored/" --include='*.c' --include='*.h' 2>/dev/null \
|
||||
| grep -v '^\s*//' | grep -v '^\s*\*' | grep -v '#define' | grep -v 'typedef' \
|
||||
| grep -v 'indicating that a fork' | head -10
|
||||
FAIL=1
|
||||
else
|
||||
echo "OK: No subprocess calls (system/popen/exec/fork) in vendored code"
|
||||
fi
|
||||
|
||||
# Network calls: not allowed in ANY vendored code. The graph-UI HTTP server
|
||||
# is first-party (src/ui/httpd.c) and audited separately by security-ui.sh.
|
||||
NETWORK_FUNCS='[^a-z_]connect\(|[^a-z_]socket\(|[^a-z_]sendto\(|[^a-z_]bind\('
|
||||
VENDORED_NETWORK=$(grep -rn -E "$NETWORK_FUNCS" "$ROOT/vendored/" --include='*.c' --include='*.h' 2>/dev/null \
|
||||
| grep -v '^\s*//' | grep -v '^\s*\*' | grep -v '#define' | grep -v 'typedef' \
|
||||
| grep -v 'sqlite3.*bind()' || true)
|
||||
if [[ -n "$VENDORED_NETWORK" ]]; then
|
||||
echo "BLOCKED: Network calls found in vendored code:"
|
||||
echo "$VENDORED_NETWORK" | head -10
|
||||
FAIL=1
|
||||
else
|
||||
echo "OK: No network calls in vendored code"
|
||||
fi
|
||||
|
||||
# dlopen/LoadLibrary: only allowed in sqlite3 (extension loading) and mimalloc (Windows APIs)
|
||||
DYNLOAD_FUNCS='dlopen\(|LoadLibrary\('
|
||||
NON_SQLITE_DYNLOAD=$(grep -rn -E "$DYNLOAD_FUNCS" "$ROOT/vendored/" --include='*.c' --include='*.h' 2>/dev/null \
|
||||
| grep -v '^\s*//' | grep -v '^\s*\*' | grep -v '#define' \
|
||||
| grep -v 'sqlite3' | grep -v 'mimalloc' || true)
|
||||
if [[ -n "$NON_SQLITE_DYNLOAD" ]]; then
|
||||
echo "BLOCKED: Dynamic library loading found outside sqlite3:"
|
||||
echo "$NON_SQLITE_DYNLOAD" | head -10
|
||||
FAIL=1
|
||||
else
|
||||
echo "OK: dlopen/LoadLibrary only in sqlite3 (blocked by authorizer at runtime)"
|
||||
fi
|
||||
|
||||
# Verify the dangerous call rules cover every vendored library.
|
||||
# Known safe: yyjson, xxhash, tre (pure computation, no OS interaction)
|
||||
# Known with exceptions: sqlite3 (dlopen), mimalloc (LoadLibrary)
|
||||
# If a new library appears, the scan above already checks it — but this ensures
|
||||
# we've consciously evaluated each library.
|
||||
KNOWN_VENDORED="mimalloc nomic sqlite3 tre xxhash yyjson"
|
||||
while IFS= read -r libdir; do
|
||||
libname=$(basename "$libdir")
|
||||
found=false
|
||||
for known in $KNOWN_VENDORED; do
|
||||
if [[ "$libname" == "$known" ]]; then
|
||||
found=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if ! $found; then
|
||||
echo "BLOCKED: vendored/${libname}/ is not in the known vendored library list"
|
||||
echo " Evaluate it for dangerous calls, then add to KNOWN_VENDORED in this script."
|
||||
FAIL=1
|
||||
fi
|
||||
done < <(find "$ROOT/vendored" -mindepth 1 -maxdepth 1 -type d | sort)
|
||||
|
||||
# Also scan tree-sitter grammars (internal/cbm/vendored/) — 650MB, 20M lines.
|
||||
# Use fast fixed-string grep (-F) for each pattern to avoid slow regex on huge codebase.
|
||||
if [[ -d "$ROOT/internal/cbm/vendored" ]]; then
|
||||
GRAMMAR_FAIL=false
|
||||
for pattern in 'system(' 'popen(' 'execl(' 'execv(' 'fork(' 'connect(' 'socket(' 'sendto(' 'dlopen(' 'LoadLibrary('; do
|
||||
HITS=$(grep -rl -F "$pattern" "$ROOT/internal/cbm/vendored/" --include='*.c' --include='*.h' 2>/dev/null | head -3 || true)
|
||||
if [[ -n "$HITS" ]]; then
|
||||
echo "BLOCKED: '$pattern' found in vendored grammars:"
|
||||
echo "$HITS" | sed 's|.*/vendored/| vendored/|'
|
||||
GRAMMAR_FAIL=true
|
||||
fi
|
||||
done
|
||||
if $GRAMMAR_FAIL; then
|
||||
FAIL=1
|
||||
else
|
||||
echo "OK: No dangerous calls in vendored tree-sitter grammars"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${1:-}" == "--update" ]]; then
|
||||
echo ""
|
||||
echo "Updating checksums..."
|
||||
find "$ROOT/vendored" -type f \( -name '*.c' -o -name '*.h' \) | sort | while IFS= read -r f; do
|
||||
$SHA_CMD "$f"
|
||||
done > "$CHECKSUMS"
|
||||
echo "Updated: $CHECKSUMS ($(wc -l < "$CHECKSUMS" | tr -d ' ') files)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ $FAIL -ne 0 ]]; then
|
||||
echo ""
|
||||
echo "=== VENDORED INTEGRITY CHECK FAILED ==="
|
||||
echo "A vendored file has been modified. If this is intentional (upgrade),"
|
||||
echo "run: scripts/security-vendored.sh --update"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $UNEXPECTED -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "WARNING: New vendored files not in checksums. Run --update if intentional."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Vendored integrity check passed ==="
|
||||
@@ -0,0 +1,326 @@
|
||||
# codebase-memory-mcp setup script (Windows)
|
||||
# Default: download pre-built native Windows binary
|
||||
# -FromSource: build from source inside WSL (requires Go + gcc in WSL)
|
||||
|
||||
param(
|
||||
[switch]$FromSource,
|
||||
[switch]$Help
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$Repo = "DeusData/codebase-memory-mcp"
|
||||
$BinaryName = "codebase-memory-mcp"
|
||||
$InstallDir = Join-Path $env:LOCALAPPDATA "codebase-memory-mcp"
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
function Write-Ok($msg) { Write-Host " $msg" -ForegroundColor Green }
|
||||
function Write-Fail($msg) { Write-Host " $msg" -ForegroundColor Red }
|
||||
function Write-Warn($msg) { Write-Host " $msg" -ForegroundColor Yellow }
|
||||
|
||||
function Read-SettingsJson($Path) {
|
||||
# PS5.1-compatible: ConvertFrom-Json returns PSCustomObject, not Hashtable.
|
||||
# We convert to ordered hashtable manually.
|
||||
if (-not (Test-Path $Path)) {
|
||||
return @{}
|
||||
}
|
||||
$raw = Get-Content $Path -Raw
|
||||
if (-not $raw -or $raw.Trim() -eq "") {
|
||||
return @{}
|
||||
}
|
||||
$obj = $raw | ConvertFrom-Json
|
||||
$ht = [ordered]@{}
|
||||
foreach ($prop in $obj.PSObject.Properties) {
|
||||
if ($prop.Value -is [System.Management.Automation.PSCustomObject]) {
|
||||
$inner = [ordered]@{}
|
||||
foreach ($p in $prop.Value.PSObject.Properties) {
|
||||
$inner[$p.Name] = $p.Value
|
||||
}
|
||||
$ht[$prop.Name] = $inner
|
||||
} else {
|
||||
$ht[$prop.Name] = $prop.Value
|
||||
}
|
||||
}
|
||||
return $ht
|
||||
}
|
||||
|
||||
function Write-SettingsJson($Path, $Settings) {
|
||||
# Back up existing file before writing
|
||||
if (Test-Path $Path) {
|
||||
Copy-Item $Path "$Path.bak" -Force
|
||||
}
|
||||
$Settings | ConvertTo-Json -Depth 10 | Set-Content $Path -Encoding UTF8
|
||||
}
|
||||
|
||||
function Configure-ClaudeCode($McpConfig) {
|
||||
Write-Host ""
|
||||
$answer = Read-Host "Configure Claude Code to use codebase-memory-mcp? [y/N]"
|
||||
|
||||
if ($answer -match '^[Yy]$') {
|
||||
$settingsPath = Join-Path $env:USERPROFILE ".claude\settings.json"
|
||||
$settingsDir = Split-Path $settingsPath -Parent
|
||||
|
||||
if (-not (Test-Path $settingsDir)) {
|
||||
New-Item -ItemType Directory -Path $settingsDir -Force | Out-Null
|
||||
}
|
||||
|
||||
$settings = Read-SettingsJson $settingsPath
|
||||
|
||||
if (-not $settings.Contains("mcpServers")) {
|
||||
$settings["mcpServers"] = [ordered]@{}
|
||||
}
|
||||
|
||||
$settings["mcpServers"]["codebase-memory-mcp"] = $McpConfig
|
||||
Write-SettingsJson $settingsPath $settings
|
||||
Write-Ok "Updated $settingsPath"
|
||||
} else {
|
||||
Write-Host ""
|
||||
Write-Host " Add this to your .mcp.json or %USERPROFILE%\.claude\settings.json:" -ForegroundColor White
|
||||
Write-Host ""
|
||||
$snippet = @{ mcpServers = @{ "codebase-memory-mcp" = $McpConfig } }
|
||||
$snippet | ConvertTo-Json -Depth 10 | Write-Host
|
||||
}
|
||||
}
|
||||
|
||||
function Test-WSL {
|
||||
try {
|
||||
$null = wsl.exe --status 2>&1
|
||||
if ($LASTEXITCODE -ne 0) { return $false }
|
||||
return $true
|
||||
} catch {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
function Get-WSLDistro {
|
||||
$output = wsl.exe -l -v 2>&1 | Out-String
|
||||
$lines = $output -split "`n" | Where-Object { $_ -match '\S' } | Select-Object -Skip 1
|
||||
foreach ($line in $lines) {
|
||||
$clean = $line -replace '\x00', '' -replace '^\s+', ''
|
||||
if ($clean -match '^\*?\s*(\S+)\s+') {
|
||||
$name = $Matches[1]
|
||||
if ($name -ne "NAME" -and $name -ne "") {
|
||||
return $name
|
||||
}
|
||||
}
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Invoke-WSL {
|
||||
param([string]$Command)
|
||||
$result = wsl.exe -- bash -c $Command 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "WSL command failed: $Command`n$result"
|
||||
}
|
||||
return $result
|
||||
}
|
||||
|
||||
# --- Main ---
|
||||
|
||||
if ($Help) {
|
||||
Write-Host ""
|
||||
Write-Host "Usage: .\setup-windows.ps1 [-FromSource] [-Help]"
|
||||
Write-Host ""
|
||||
Write-Host " Default: Download pre-built Windows binary"
|
||||
Write-Host " -FromSource: Build from source inside WSL (requires Go 1.23+ and gcc in WSL)"
|
||||
Write-Host ""
|
||||
exit 0
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "codebase-memory-mcp installer (Windows)" -ForegroundColor White
|
||||
Write-Host ""
|
||||
|
||||
if ($FromSource) {
|
||||
# --- Build from source via WSL ---
|
||||
Write-Host "Checking WSL2 (required for building from source)..." -ForegroundColor White
|
||||
|
||||
if (-not (Test-WSL)) {
|
||||
Write-Fail "WSL2 is not available. Required for building from source (CGO needs gcc)."
|
||||
Write-Host ""
|
||||
Write-Host " Install WSL2:" -ForegroundColor Yellow
|
||||
Write-Host " wsl --install -d Ubuntu"
|
||||
Write-Host " Then restart your computer and run this script again."
|
||||
Write-Host ""
|
||||
Write-Host " Alternatively, download a pre-built binary (no WSL needed):" -ForegroundColor Yellow
|
||||
Write-Host " .\setup-windows.ps1"
|
||||
exit 1
|
||||
}
|
||||
Write-Ok "WSL2 is available"
|
||||
|
||||
$distro = Get-WSLDistro
|
||||
if (-not $distro) {
|
||||
Write-Fail "No WSL Linux distribution found."
|
||||
Write-Host ""
|
||||
Write-Host " Install Ubuntu:" -ForegroundColor Yellow
|
||||
Write-Host " wsl --install -d Ubuntu"
|
||||
exit 1
|
||||
}
|
||||
Write-Ok "WSL distro: $distro"
|
||||
|
||||
$wslUser = (Invoke-WSL "whoami").Trim()
|
||||
Write-Ok "WSL user: $wslUser"
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Checking prerequisites inside WSL..." -ForegroundColor White
|
||||
|
||||
# Check for gcc
|
||||
try {
|
||||
Invoke-WSL "command -v gcc" | Out-Null
|
||||
Write-Ok "gcc found"
|
||||
} catch {
|
||||
Write-Warn "gcc not found. Installing build-essential..."
|
||||
Invoke-WSL "sudo apt-get update && sudo apt-get install -y build-essential"
|
||||
}
|
||||
|
||||
# Check for Go
|
||||
try {
|
||||
$goVersion = Invoke-WSL "go version"
|
||||
Write-Ok "Go: $goVersion"
|
||||
} catch {
|
||||
Write-Fail "Go not found in WSL."
|
||||
Write-Host ""
|
||||
Write-Host " Install Go 1.23+ inside WSL:" -ForegroundColor Yellow
|
||||
Write-Host " See https://go.dev/dl/ for Linux amd64 tarball"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Check for git
|
||||
try {
|
||||
Invoke-WSL "command -v git" | Out-Null
|
||||
Write-Ok "git found"
|
||||
} catch {
|
||||
Write-Fail "git not found in WSL. Install with: sudo apt install git"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Clone or update
|
||||
Write-Host ""
|
||||
$sourceDir = "/home/$wslUser/.local/share/codebase-memory-mcp"
|
||||
try {
|
||||
Invoke-WSL "test -d $sourceDir/.git" | Out-Null
|
||||
Write-Host "Updating source..." -ForegroundColor White
|
||||
Invoke-WSL "git -C $sourceDir pull --ff-only"
|
||||
} catch {
|
||||
Write-Host "Cloning repository..." -ForegroundColor White
|
||||
Invoke-WSL "mkdir -p /home/$wslUser/.local/share && git clone https://github.com/$Repo.git $sourceDir"
|
||||
}
|
||||
Write-Ok "Source at $sourceDir"
|
||||
|
||||
# Build
|
||||
Write-Host ""
|
||||
Write-Host "Building binary (this may take a minute)..." -ForegroundColor White
|
||||
$wslBinaryPath = "/home/$wslUser/.local/bin/$BinaryName"
|
||||
Invoke-WSL "mkdir -p /home/$wslUser/.local/bin && cd $sourceDir && scripts/build.sh && cp build/c/codebase-memory-mcp $wslBinaryPath"
|
||||
Write-Ok "Built to $wslBinaryPath (inside WSL)"
|
||||
|
||||
# Verify
|
||||
try {
|
||||
Invoke-WSL "test -x $wslBinaryPath" | Out-Null
|
||||
Write-Ok "Binary is executable"
|
||||
} catch {
|
||||
Write-Fail "Binary at $wslBinaryPath is not executable"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Configure — WSL binary needs wsl.exe wrapper
|
||||
$mcpConfig = [ordered]@{
|
||||
type = "stdio"
|
||||
command = "wsl.exe"
|
||||
args = @("-d", $distro, "--", $wslBinaryPath)
|
||||
}
|
||||
|
||||
Configure-ClaudeCode $mcpConfig
|
||||
|
||||
Write-Host ""
|
||||
Write-Ok "Done! Restart Claude Code and verify with /mcp"
|
||||
Write-Host ""
|
||||
Write-Host " To uninstall:" -ForegroundColor White
|
||||
Write-Host " wsl.exe -- rm $wslBinaryPath"
|
||||
Write-Host " wsl.exe -- rm -rf $sourceDir"
|
||||
Write-Host " wsl.exe -- rm -rf ~/.cache/codebase-memory-mcp/"
|
||||
|
||||
} else {
|
||||
# --- Download pre-built native Windows binary ---
|
||||
Write-Host "Fetching latest release..." -ForegroundColor White
|
||||
|
||||
$releaseUrl = "https://api.github.com/repos/$Repo/releases/latest"
|
||||
$release = Invoke-RestMethod -Uri $releaseUrl -Headers @{ "User-Agent" = "codebase-memory-mcp-setup" }
|
||||
$tag = $release.tag_name
|
||||
|
||||
if (-not $tag) {
|
||||
Write-Fail "Could not determine latest release."
|
||||
Write-Host " Check: https://github.com/$Repo/releases"
|
||||
exit 1
|
||||
}
|
||||
Write-Ok "Latest release: $tag"
|
||||
|
||||
$asset = "codebase-memory-mcp-windows-amd64.zip"
|
||||
$downloadUrl = "https://github.com/$Repo/releases/download/$tag/$asset"
|
||||
|
||||
Write-Host "Downloading $asset..." -ForegroundColor White
|
||||
|
||||
# Create install directory
|
||||
if (-not (Test-Path $InstallDir)) {
|
||||
New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null
|
||||
}
|
||||
|
||||
$tmpZip = Join-Path $env:TEMP $asset
|
||||
Invoke-WebRequest -Uri $downloadUrl -OutFile $tmpZip -UseBasicParsing
|
||||
|
||||
# Extract
|
||||
Expand-Archive -Path $tmpZip -DestinationPath $InstallDir -Force
|
||||
Remove-Item $tmpZip -Force
|
||||
|
||||
$binaryPath = Join-Path $InstallDir "$BinaryName.exe"
|
||||
|
||||
if (-not (Test-Path $binaryPath)) {
|
||||
Write-Fail "Binary not found at $binaryPath after extraction"
|
||||
exit 1
|
||||
}
|
||||
Write-Ok "Installed to $binaryPath"
|
||||
|
||||
# Verify binary runs
|
||||
try {
|
||||
$verOut = & $binaryPath --version 2>&1
|
||||
Write-Ok "Version: $verOut"
|
||||
} catch {
|
||||
Write-Warn "Could not verify binary version (may still work)"
|
||||
}
|
||||
|
||||
# SmartScreen note
|
||||
Write-Host ""
|
||||
Write-Warn "Windows SmartScreen may show a warning when the binary runs for the first time."
|
||||
Write-Host " This is normal for unsigned open-source binaries." -ForegroundColor Yellow
|
||||
Write-Host " Click 'More info' then 'Run anyway' to proceed." -ForegroundColor Yellow
|
||||
Write-Host " Verify checksums at: https://github.com/$Repo/releases" -ForegroundColor Yellow
|
||||
|
||||
# Check if install dir is on PATH
|
||||
$userPath = [Environment]::GetEnvironmentVariable("Path", "User")
|
||||
if ($userPath -notlike "*$InstallDir*") {
|
||||
Write-Host ""
|
||||
Write-Warn "$InstallDir is not on your PATH."
|
||||
$addPath = Read-Host " Add it to your user PATH? [y/N]"
|
||||
if ($addPath -match '^[Yy]$') {
|
||||
[Environment]::SetEnvironmentVariable("Path", "$userPath;$InstallDir", "User")
|
||||
Write-Ok "Added to user PATH (restart your terminal to take effect)"
|
||||
}
|
||||
}
|
||||
|
||||
# Configure Claude Code
|
||||
$mcpConfig = [ordered]@{
|
||||
type = "stdio"
|
||||
command = $binaryPath
|
||||
}
|
||||
|
||||
Configure-ClaudeCode $mcpConfig
|
||||
|
||||
Write-Host ""
|
||||
Write-Ok "Done! Restart Claude Code and verify with /mcp"
|
||||
Write-Host ""
|
||||
Write-Host " To uninstall:" -ForegroundColor White
|
||||
Write-Host " Remove-Item -Recurse -Force '$InstallDir'"
|
||||
Write-Host " Remove-Item -Recurse -Force `"$env:LOCALAPPDATA\codebase-memory-mcp`" # graph database"
|
||||
}
|
||||
Executable
+335
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# codebase-memory-mcp setup script (macOS + Linux)
|
||||
# Default: download pre-built binary from GitHub Release
|
||||
# --from-source: build from source (requires Go + C compiler)
|
||||
|
||||
REPO="DeusData/codebase-memory-mcp"
|
||||
INSTALL_DIR="$HOME/.local/bin"
|
||||
BINARY_NAME="codebase-memory-mcp"
|
||||
SOURCE_DIR="$HOME/.local/share/codebase-memory-mcp"
|
||||
CLEANUP_DIR="" # set by download_binary for EXIT trap
|
||||
|
||||
# --- Colors ---
|
||||
|
||||
if [ -t 1 ] && command -v tput &>/dev/null; then
|
||||
GREEN=$(tput setaf 2)
|
||||
RED=$(tput setaf 1)
|
||||
YELLOW=$(tput setaf 3)
|
||||
BOLD=$(tput bold)
|
||||
RESET=$(tput sgr0)
|
||||
else
|
||||
GREEN=""
|
||||
RED=""
|
||||
YELLOW=""
|
||||
BOLD=""
|
||||
RESET=""
|
||||
fi
|
||||
|
||||
ok() { echo "${GREEN}✓${RESET} $*"; }
|
||||
fail() { echo "${RED}✗${RESET} $*"; }
|
||||
warn() { echo "${YELLOW}⚠${RESET} $*"; }
|
||||
info() { echo " $*"; }
|
||||
|
||||
die() { fail "$@"; exit 1; }
|
||||
|
||||
# --- Argument parsing ---
|
||||
|
||||
FROM_SOURCE=false
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--from-source) FROM_SOURCE=true ;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [--from-source]"
|
||||
echo ""
|
||||
echo " Default: Download pre-built binary from GitHub Release"
|
||||
echo " --from-source: Clone and build from source (requires Go 1.23+ and a C compiler)"
|
||||
exit 0
|
||||
;;
|
||||
*) die "Unknown argument: $arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# --- Platform detection ---
|
||||
|
||||
detect_platform() {
|
||||
local os arch
|
||||
os=$(uname -s)
|
||||
arch=$(uname -m)
|
||||
|
||||
case "$os" in
|
||||
Darwin) os="darwin" ;;
|
||||
Linux) os="linux" ;;
|
||||
*) die "Unsupported OS: $os. Use WSL2 on Windows." ;;
|
||||
esac
|
||||
|
||||
case "$arch" in
|
||||
arm64|aarch64) arch="arm64" ;;
|
||||
x86_64|amd64)
|
||||
# On macOS, uname -m returns x86_64 under Rosetta even on Apple Silicon.
|
||||
# Check the actual hardware to pick the right binary.
|
||||
if [ "$os" = "darwin" ] && sysctl -n hw.optional.arm64 2>/dev/null | grep -q '1'; then
|
||||
arch="arm64"
|
||||
else
|
||||
arch="amd64"
|
||||
fi
|
||||
;;
|
||||
*) die "Unsupported architecture: $arch" ;;
|
||||
esac
|
||||
|
||||
echo "${os}-${arch}"
|
||||
}
|
||||
|
||||
# --- Prerequisite checks ---
|
||||
|
||||
check_download_tool() {
|
||||
if command -v curl &>/dev/null; then
|
||||
echo "curl"
|
||||
elif command -v wget &>/dev/null; then
|
||||
echo "wget"
|
||||
else
|
||||
die "Neither curl nor wget found. Install one and retry."
|
||||
fi
|
||||
}
|
||||
|
||||
check_go_version() {
|
||||
if ! command -v go &>/dev/null; then
|
||||
die "Go not found. Install Go 1.23+ from https://go.dev/dl/"
|
||||
fi
|
||||
|
||||
local version
|
||||
version=$(go version | grep -oE 'go[0-9]+\.[0-9]+' | head -1)
|
||||
local major minor
|
||||
major=$(echo "$version" | grep -oE '[0-9]+' | head -1)
|
||||
minor=$(echo "$version" | grep -oE '[0-9]+' | sed -n '2p')
|
||||
|
||||
if [ "$major" -lt 1 ] || { [ "$major" -eq 1 ] && [ "$minor" -lt 23 ]; }; then
|
||||
die "Go $major.$minor found, but 1.23+ is required. Update from https://go.dev/dl/"
|
||||
fi
|
||||
|
||||
ok "Go $major.$minor"
|
||||
}
|
||||
|
||||
check_c_compiler() {
|
||||
if command -v cc &>/dev/null || command -v gcc &>/dev/null || command -v clang &>/dev/null; then
|
||||
ok "C compiler found"
|
||||
return
|
||||
fi
|
||||
|
||||
local os
|
||||
os=$(uname -s)
|
||||
if [ "$os" = "Darwin" ]; then
|
||||
die "No C compiler found. Run: xcode-select --install"
|
||||
else
|
||||
die "No C compiler found. Install build-essential (Debian/Ubuntu) or gcc (Fedora/RHEL)"
|
||||
fi
|
||||
}
|
||||
|
||||
check_git() {
|
||||
if ! command -v git &>/dev/null; then
|
||||
die "Git not found. Install git and retry."
|
||||
fi
|
||||
ok "Git found"
|
||||
}
|
||||
|
||||
# --- Download binary ---
|
||||
|
||||
fetch() {
|
||||
local url="$1" tool="$2"
|
||||
if [ "$tool" = "curl" ]; then
|
||||
curl -fsSL "$url"
|
||||
else
|
||||
wget -qO- "$url"
|
||||
fi
|
||||
}
|
||||
|
||||
download_binary() {
|
||||
local platform="$1" tool="$2"
|
||||
|
||||
echo ""
|
||||
echo "${BOLD}Fetching latest release...${RESET}"
|
||||
local tag
|
||||
tag=$(fetch "https://api.github.com/repos/${REPO}/releases/latest" "$tool" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"//;s/".*//')
|
||||
|
||||
if [ -z "$tag" ]; then
|
||||
die "Could not determine latest release. Check https://github.com/${REPO}/releases"
|
||||
fi
|
||||
ok "Latest release: $tag"
|
||||
|
||||
local asset="codebase-memory-mcp-${platform}.tar.gz"
|
||||
local url="https://github.com/${REPO}/releases/download/${tag}/${asset}"
|
||||
|
||||
echo "${BOLD}Downloading ${asset}...${RESET}"
|
||||
CLEANUP_DIR=$(mktemp -d)
|
||||
trap 'rm -rf "$CLEANUP_DIR"' EXIT
|
||||
local tmpdir="$CLEANUP_DIR"
|
||||
|
||||
fetch "$url" "$tool" > "${tmpdir}/${asset}"
|
||||
tar -xzf "${tmpdir}/${asset}" -C "$tmpdir"
|
||||
|
||||
mkdir -p "$INSTALL_DIR"
|
||||
mv "${tmpdir}/${BINARY_NAME}" "${INSTALL_DIR}/${BINARY_NAME}"
|
||||
chmod +x "${INSTALL_DIR}/${BINARY_NAME}"
|
||||
|
||||
ok "Installed to ${INSTALL_DIR}/${BINARY_NAME}"
|
||||
}
|
||||
|
||||
# --- Build from source ---
|
||||
|
||||
build_from_source() {
|
||||
echo ""
|
||||
echo "${BOLD}Checking prerequisites...${RESET}"
|
||||
check_go_version
|
||||
check_c_compiler
|
||||
check_git
|
||||
|
||||
echo ""
|
||||
if [ -d "$SOURCE_DIR/.git" ]; then
|
||||
echo "${BOLD}Updating source...${RESET}"
|
||||
git -C "$SOURCE_DIR" pull --ff-only
|
||||
else
|
||||
echo "${BOLD}Cloning repository...${RESET}"
|
||||
mkdir -p "$(dirname "$SOURCE_DIR")"
|
||||
git clone "https://github.com/${REPO}.git" "$SOURCE_DIR"
|
||||
fi
|
||||
ok "Source at ${SOURCE_DIR}"
|
||||
|
||||
echo ""
|
||||
echo "${BOLD}Building binary (this may take a minute)...${RESET}"
|
||||
mkdir -p "$INSTALL_DIR"
|
||||
|
||||
(cd "$SOURCE_DIR" && scripts/build.sh && cp build/c/codebase-memory-mcp "${INSTALL_DIR}/${BINARY_NAME}")
|
||||
|
||||
ok "Built and installed to ${INSTALL_DIR}/${BINARY_NAME}"
|
||||
}
|
||||
|
||||
# --- MCP auto-configuration ---
|
||||
|
||||
configure_claude() {
|
||||
echo ""
|
||||
local binary_path="${INSTALL_DIR}/${BINARY_NAME}"
|
||||
local claude_config_dir="${CLAUDE_CONFIG_DIR:-$HOME/.claude}"
|
||||
local settings_file="${claude_config_dir}/settings.json"
|
||||
|
||||
printf "%s" "${BOLD}Configure Claude Code to use codebase-memory-mcp? [y/N] ${RESET}"
|
||||
read -r answer
|
||||
if [[ ! "$answer" =~ ^[Yy]$ ]]; then
|
||||
echo ""
|
||||
info "Add this to your .mcp.json or ${claude_config_dir}/settings.json:"
|
||||
echo ""
|
||||
echo ' {'
|
||||
echo ' "mcpServers": {'
|
||||
echo ' "codebase-memory-mcp": {'
|
||||
echo ' "type": "stdio",'
|
||||
echo " \"command\": \"${binary_path}\""
|
||||
echo ' }'
|
||||
echo ' }'
|
||||
echo ' }'
|
||||
return
|
||||
fi
|
||||
|
||||
local mcp_entry
|
||||
mcp_entry=$(cat <<JSONEOF
|
||||
{"type":"stdio","command":"${binary_path}"}
|
||||
JSONEOF
|
||||
)
|
||||
|
||||
mkdir -p "$(dirname "$settings_file")"
|
||||
|
||||
if command -v jq &>/dev/null; then
|
||||
# Use jq to merge
|
||||
if [ -f "$settings_file" ]; then
|
||||
local tmp
|
||||
tmp=$(mktemp)
|
||||
jq --argjson entry "$mcp_entry" '.mcpServers["codebase-memory-mcp"] = $entry' "$settings_file" > "$tmp"
|
||||
mv "$tmp" "$settings_file"
|
||||
else
|
||||
echo "{}" | jq --argjson entry "$mcp_entry" '.mcpServers["codebase-memory-mcp"] = $entry' > "$settings_file"
|
||||
fi
|
||||
ok "Updated ${settings_file}"
|
||||
elif command -v python3 &>/dev/null; then
|
||||
python3 -c "
|
||||
import json, os
|
||||
path = os.path.expanduser('$settings_file')
|
||||
data = {}
|
||||
if os.path.exists(path):
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
data.setdefault('mcpServers', {})['codebase-memory-mcp'] = json.loads('$mcp_entry')
|
||||
with open(path, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
print()
|
||||
"
|
||||
ok "Updated ${settings_file}"
|
||||
else
|
||||
warn "Neither jq nor python3 found — cannot auto-configure."
|
||||
echo ""
|
||||
info "Add this to ${settings_file} manually:"
|
||||
echo ""
|
||||
echo ' "mcpServers": {'
|
||||
echo ' "codebase-memory-mcp": {'
|
||||
echo ' "type": "stdio",'
|
||||
echo " \"command\": \"${binary_path}\""
|
||||
echo ' }'
|
||||
echo ' }'
|
||||
fi
|
||||
}
|
||||
|
||||
# --- PATH check ---
|
||||
|
||||
check_path() {
|
||||
if [[ ":$PATH:" != *":${INSTALL_DIR}:"* ]]; then
|
||||
echo ""
|
||||
warn "${INSTALL_DIR} is not on your PATH."
|
||||
info "Add this to your shell profile (~/.bashrc, ~/.zshrc, etc.):"
|
||||
echo ""
|
||||
echo " export PATH=\"${INSTALL_DIR}:\$PATH\""
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Main ---
|
||||
|
||||
echo ""
|
||||
echo "${BOLD}codebase-memory-mcp installer${RESET}"
|
||||
echo ""
|
||||
|
||||
if [ "$FROM_SOURCE" = true ]; then
|
||||
build_from_source
|
||||
else
|
||||
platform=$(detect_platform)
|
||||
ok "Platform: ${platform}"
|
||||
tool=$(check_download_tool)
|
||||
ok "Download tool: ${tool}"
|
||||
download_binary "$platform" "$tool"
|
||||
fi
|
||||
|
||||
# Verify binary
|
||||
if [ ! -x "${INSTALL_DIR}/${BINARY_NAME}" ]; then
|
||||
die "Binary at ${INSTALL_DIR}/${BINARY_NAME} is not executable"
|
||||
fi
|
||||
|
||||
ver_output=$("${INSTALL_DIR}/${BINARY_NAME}" --version 2>&1) || true
|
||||
if [ -n "$ver_output" ]; then
|
||||
ok "$ver_output"
|
||||
else
|
||||
ok "Binary is executable"
|
||||
fi
|
||||
|
||||
configure_claude
|
||||
check_path
|
||||
|
||||
# --- Git hooks ---
|
||||
# If run from inside the repo, activate tracked hooks
|
||||
if [ -d "scripts/hooks" ] && git rev-parse --git-dir &>/dev/null; then
|
||||
git config core.hooksPath scripts/hooks
|
||||
ok "Git hooks activated (scripts/hooks/)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
ok "Done! Restart Claude Code and verify with /mcp"
|
||||
echo ""
|
||||
info "To uninstall:"
|
||||
info " rm ${INSTALL_DIR}/${BINARY_NAME}"
|
||||
info " rm -rf ${SOURCE_DIR} # if built from source"
|
||||
info " rm -rf ~/.cache/codebase-memory-mcp/ # graph database"
|
||||
Executable
+1069
File diff suppressed because it is too large
Load Diff
Executable
+1938
File diff suppressed because it is too large
Load Diff
Executable
+491
@@ -0,0 +1,491 @@
|
||||
#!/usr/bin/env bash
|
||||
# soak-test.sh — Endurance test for codebase-memory-mcp.
|
||||
#
|
||||
# Runs compressed workload cycles: queries, file mutations, reindexes, idle periods.
|
||||
# Reads diagnostics from /tmp/cbm-diagnostics-<pid>.json (requires CBM_DIAGNOSTICS=1).
|
||||
# Outputs metrics to soak-results/ and exits 0 (pass) or 1 (fail).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/soak-test.sh <binary> <duration_minutes> [--skip-crash-test]
|
||||
#
|
||||
# Tiers:
|
||||
# 10 min = quick soak (CI gate)
|
||||
# 15 min = ASan soak (leak detection)
|
||||
# 240 min = nightly (compressed 4h = ~5 days real usage)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BINARY="${1:?Usage: soak-test.sh <binary> <duration_minutes>}"
|
||||
DURATION_MIN="${2:?Usage: soak-test.sh <binary> <duration_minutes>}"
|
||||
SKIP_CRASH="${3:-}"
|
||||
BINARY=$(cd "$(dirname "$BINARY")" && pwd)/$(basename "$BINARY")
|
||||
|
||||
# Soak mode selector.
|
||||
# default = original mixed workload (queries + mutations + periodic reindex
|
||||
# + crash-recovery). Unchanged from before this env var existed.
|
||||
# query-leak = #581 detector. After the initial index, NEVER reindex and NEVER
|
||||
# mutate files, so the mimalloc page-return path (cbm_mem_collect,
|
||||
# triggered by index_repository) is never invoked and cannot sweep
|
||||
# a query-only leak. Phase 3 then hammers a variety of READ tools
|
||||
# (search_graph / query_graph / trace_path / get_code_snippet /
|
||||
# search_code) to exercise the query-only store-open + WAL + alloc
|
||||
# paths the bug report implicates. The RSS slope/ratio/ceiling
|
||||
# analysis below is the leak detector. The crash-recovery phase is
|
||||
# skipped in this mode because it reindexes (which would mask #581).
|
||||
CBM_SOAK_MODE="${CBM_SOAK_MODE:-default}"
|
||||
|
||||
RESULTS_DIR="${RESULTS_DIR:-soak-results}"
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
METRICS_CSV="$RESULTS_DIR/metrics.csv"
|
||||
LATENCY_CSV="$RESULTS_DIR/latency.csv"
|
||||
SUMMARY="$RESULTS_DIR/summary.txt"
|
||||
|
||||
echo "timestamp,uptime_s,rss_bytes,heap_committed,fd_count,query_count,query_max_us" > "$METRICS_CSV"
|
||||
echo "timestamp,tool,duration_ms,exit_code" > "$LATENCY_CSV"
|
||||
> "$SUMMARY"
|
||||
|
||||
DURATION_S=$((DURATION_MIN * 60))
|
||||
|
||||
echo "=== soak-test: binary=$BINARY duration=${DURATION_MIN}m mode=${CBM_SOAK_MODE} ==="
|
||||
|
||||
# ── Helper: generate realistic test project (~200 files) ─────────
|
||||
|
||||
SOAK_PROJECT=$(mktemp -d)
|
||||
|
||||
generate_project() {
|
||||
local root="$1"
|
||||
# Python package (80 files)
|
||||
for i in $(seq 1 20); do
|
||||
local pkg="$root/src/pkg_${i}"
|
||||
mkdir -p "$pkg"
|
||||
cat > "$pkg/__init__.py" << PYEOF
|
||||
from .handlers import handle_${i}
|
||||
from .models import Model${i}
|
||||
PYEOF
|
||||
cat > "$pkg/handlers.py" << PYEOF
|
||||
from .models import Model${i}
|
||||
from .utils import validate_${i}, transform_${i}
|
||||
|
||||
def handle_${i}(request):
|
||||
data = Model${i}.from_request(request)
|
||||
if not validate_${i}(data):
|
||||
return {"error": "invalid"}
|
||||
return transform_${i}(data)
|
||||
|
||||
def process_batch_${i}(items):
|
||||
return [handle_${i}(item) for item in items]
|
||||
PYEOF
|
||||
cat > "$pkg/models.py" << PYEOF
|
||||
class Model${i}:
|
||||
def __init__(self, name, value):
|
||||
self.name = name
|
||||
self.value = value
|
||||
|
||||
@classmethod
|
||||
def from_request(cls, req):
|
||||
return cls(req.get("name", ""), req.get("value", 0))
|
||||
|
||||
def to_dict(self):
|
||||
return {"name": self.name, "value": self.value}
|
||||
PYEOF
|
||||
cat > "$pkg/utils.py" << PYEOF
|
||||
def validate_${i}(data):
|
||||
return data is not None and hasattr(data, 'name')
|
||||
|
||||
def transform_${i}(data):
|
||||
return {"result": data.name.upper(), "score": data.value * ${i}}
|
||||
PYEOF
|
||||
done
|
||||
|
||||
# Go package (40 files)
|
||||
mkdir -p "$root/internal/api" "$root/internal/store" "$root/cmd"
|
||||
for i in $(seq 1 20); do
|
||||
cat > "$root/internal/api/handler_${i}.go" << GOEOF
|
||||
package api
|
||||
|
||||
import "fmt"
|
||||
|
||||
func HandleRoute${i}(path string) (string, error) {
|
||||
result := ProcessData${i}(path)
|
||||
return fmt.Sprintf("route_%d: %s", ${i}, result), nil
|
||||
}
|
||||
|
||||
func ProcessData${i}(input string) string {
|
||||
return fmt.Sprintf("processed_%d_%s", ${i}, input)
|
||||
}
|
||||
GOEOF
|
||||
cat > "$root/internal/store/repo_${i}.go" << GOEOF
|
||||
package store
|
||||
|
||||
type Entity${i} struct {
|
||||
ID int
|
||||
Name string
|
||||
Data map[string]interface{}
|
||||
}
|
||||
|
||||
func FindEntity${i}(id int) (*Entity${i}, error) {
|
||||
return &Entity${i}{ID: id, Name: "entity"}, nil
|
||||
}
|
||||
|
||||
func SaveEntity${i}(e *Entity${i}) error {
|
||||
return nil
|
||||
}
|
||||
GOEOF
|
||||
done
|
||||
|
||||
# TypeScript (40 files)
|
||||
mkdir -p "$root/frontend/src/components" "$root/frontend/src/hooks"
|
||||
for i in $(seq 1 20); do
|
||||
cat > "$root/frontend/src/components/Component${i}.tsx" << TSEOF
|
||||
import React from 'react';
|
||||
import { useData${i} } from '../hooks/useData${i}';
|
||||
|
||||
interface Props${i} { id: number; label: string; }
|
||||
|
||||
export const Component${i}: React.FC<Props${i}> = ({ id, label }) => {
|
||||
const { data, loading } = useData${i}(id);
|
||||
if (loading) return <div>Loading...</div>;
|
||||
return <div className="comp-${i}">{label}: {JSON.stringify(data)}</div>;
|
||||
};
|
||||
TSEOF
|
||||
cat > "$root/frontend/src/hooks/useData${i}.ts" << TSEOF
|
||||
import { useState, useEffect } from 'react';
|
||||
|
||||
export function useData${i}(id: number) {
|
||||
const [data, setData] = useState(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
useEffect(() => {
|
||||
fetch('/api/data/${i}/' + id)
|
||||
.then(r => r.json())
|
||||
.then(d => { setData(d); setLoading(false); });
|
||||
}, [id]);
|
||||
return { data, loading };
|
||||
}
|
||||
TSEOF
|
||||
done
|
||||
|
||||
# Config files
|
||||
cat > "$root/config.yaml" << 'YAMLEOF'
|
||||
database:
|
||||
host: localhost
|
||||
port: 5432
|
||||
pool_size: 10
|
||||
server:
|
||||
workers: 4
|
||||
timeout: 30
|
||||
YAMLEOF
|
||||
cat > "$root/Dockerfile" << 'DEOF'
|
||||
FROM python:3.11-slim
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
RUN pip install -r requirements.txt
|
||||
CMD ["python", "-m", "src.main"]
|
||||
DEOF
|
||||
}
|
||||
|
||||
echo "Generating test project (~200 files)..."
|
||||
generate_project "$SOAK_PROJECT"
|
||||
|
||||
# Init git repo (required for watcher)
|
||||
git -C "$SOAK_PROJECT" init -q 2>/dev/null
|
||||
git -C "$SOAK_PROJECT" add -A 2>/dev/null
|
||||
git -C "$SOAK_PROJECT" -c user.email=test@test -c user.name=test commit -q -m "init" 2>/dev/null
|
||||
FILE_COUNT=$(find "$SOAK_PROJECT" -type f | wc -l | tr -d ' ')
|
||||
echo "OK: $FILE_COUNT files in test project"
|
||||
|
||||
# ── Helper: run CLI tool call and record latency ─────────────────
|
||||
|
||||
# Query ID counter
|
||||
QUERY_ID=1
|
||||
|
||||
# Send a JSON-RPC tool call to the running server via its stdin pipe.
|
||||
# Reads response from server stdout. Records latency.
|
||||
mcp_call() {
|
||||
local tool="$1"
|
||||
local args="$2"
|
||||
local id=$QUERY_ID
|
||||
QUERY_ID=$((QUERY_ID + 1))
|
||||
|
||||
local req="{\"jsonrpc\":\"2.0\",\"id\":$id,\"method\":\"tools/call\",\"params\":{\"name\":\"$tool\",\"arguments\":$args}}"
|
||||
local t0
|
||||
t0=$(python3 -c "import time; print(int(time.time()*1000))")
|
||||
|
||||
# Send request to server stdin
|
||||
echo "$req" >&3
|
||||
|
||||
# Read response (wait up to 30s)
|
||||
local resp=""
|
||||
if read -t 30 resp <&4 2>/dev/null; then
|
||||
local t1
|
||||
t1=$(python3 -c "import time; print(int(time.time()*1000))")
|
||||
local dur=$((t1 - t0))
|
||||
echo "$(date +%s),$tool,$dur,0" >> "$LATENCY_CSV"
|
||||
else
|
||||
local t1
|
||||
t1=$(python3 -c "import time; print(int(time.time()*1000))")
|
||||
local dur=$((t1 - t0))
|
||||
echo "$(date +%s),$tool,$dur,1" >> "$LATENCY_CSV"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Helper: collect diagnostics snapshot ─────────────────────────
|
||||
|
||||
collect_snapshot() {
|
||||
local diag_file="/tmp/cbm-diagnostics-${SERVER_PID}.json"
|
||||
if [ -f "$diag_file" ]; then
|
||||
python3 -c "
|
||||
import json, time
|
||||
d = json.load(open('$diag_file'))
|
||||
# Use heap_committed if available, otherwise RSS (mimalloc may report 0 for committed)
|
||||
mem = d.get('heap_committed_bytes', 0)
|
||||
if mem == 0: mem = d.get('rss_bytes', 0)
|
||||
print(f\"{int(time.time())},{d.get('uptime_s',0)},{d.get('rss_bytes',0)},{mem},{d.get('fd_count',0)},{d.get('query_count',0)},{d.get('query_max_us',0)}\")
|
||||
" 2>/dev/null >> "$METRICS_CSV"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Phase 1: Start MCP server with diagnostics ──────────────────
|
||||
|
||||
echo "--- Phase 1: start server ---"
|
||||
# Bidirectional pipes: fd3 = server stdin (write), fd4 = server stdout (read)
|
||||
SERVER_IN=$(mktemp -u).in
|
||||
SERVER_OUT=$(mktemp -u).out
|
||||
mkfifo "$SERVER_IN" "$SERVER_OUT"
|
||||
|
||||
CBM_DIAGNOSTICS=1 "$BINARY" < "$SERVER_IN" > "$SERVER_OUT" 2>"$RESULTS_DIR/server-stderr.log" &
|
||||
SERVER_PID=$!
|
||||
|
||||
# Open fds AFTER server starts (otherwise fifo blocks)
|
||||
exec 3>"$SERVER_IN" # write to server stdin
|
||||
exec 4<"$SERVER_OUT" # read from server stdout
|
||||
sleep 3
|
||||
|
||||
if ! kill -0 "$SERVER_PID" 2>/dev/null; then
|
||||
echo "FAIL: server did not start"
|
||||
exec 3>&- 4<&-
|
||||
rm -f "$SERVER_IN" "$SERVER_OUT"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: server running (pid=$SERVER_PID)"
|
||||
|
||||
# Send initialize handshake
|
||||
echo '{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"capabilities":{}}}' >&3
|
||||
read -t 10 INIT_RESP <&4 || true
|
||||
|
||||
# ── Phase 2: Initial index ───────────────────────────────────────
|
||||
|
||||
echo "--- Phase 2: initial index ---"
|
||||
mcp_call index_repository "{\"repo_path\":\"$SOAK_PROJECT\"}"
|
||||
sleep 6 # wait for diagnostics write
|
||||
collect_snapshot
|
||||
|
||||
# Derive project name (same logic as cbm_project_name_from_path)
|
||||
PROJ_NAME=$(echo "$SOAK_PROJECT" | sed 's|^/||; s|/|-|g')
|
||||
|
||||
DIAG_FILE="/tmp/cbm-diagnostics-${SERVER_PID}.json"
|
||||
BASELINE_RSS=$(cat "$DIAG_FILE" 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('rss_bytes',0))" 2>/dev/null || echo "0")
|
||||
BASELINE_FDS=$(cat "$DIAG_FILE" 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('fd_count',0))" 2>/dev/null || echo "0")
|
||||
echo "OK: baseline RSS=${BASELINE_RSS} FDs=${BASELINE_FDS}"
|
||||
|
||||
# ── Phase 3: Compressed workload loop ────────────────────────────
|
||||
|
||||
echo "--- Phase 3: workload loop (${DURATION_MIN}m) ---"
|
||||
START_TIME=$(date +%s)
|
||||
END_TIME=$((START_TIME + DURATION_S))
|
||||
CYCLE=0
|
||||
LAST_MUTATE=0
|
||||
LAST_REINDEX=0
|
||||
|
||||
while [ "$(date +%s)" -lt "$END_TIME" ]; do
|
||||
NOW=$(date +%s)
|
||||
CYCLE=$((CYCLE + 1))
|
||||
|
||||
if [ "$CBM_SOAK_MODE" = "query-leak" ]; then
|
||||
# ── #581 query-only leak mode ────────────────────────────────
|
||||
# Pure read-query hammering: no mutation, no reindex — so
|
||||
# cbm_mem_collect (mimalloc page return) is NEVER triggered and
|
||||
# cannot sweep a query-only leak. Hammer a VARIETY of read tools to
|
||||
# exercise the store-open + WAL + alloc paths the report implicates.
|
||||
mcp_call search_graph "{\"project\":\"$PROJ_NAME\",\"name_pattern\":\".*Handle.*\"}"
|
||||
mcp_call query_graph "{\"project\":\"$PROJ_NAME\",\"query\":\"MATCH (n) RETURN n.name LIMIT 25\"}"
|
||||
mcp_call trace_path "{\"project\":\"$PROJ_NAME\",\"function_name\":\"handle_1\",\"direction\":\"both\"}"
|
||||
mcp_call get_code_snippet "{\"project\":\"$PROJ_NAME\",\"qualified_name\":\"handle_1\"}"
|
||||
mcp_call search_code "{\"project\":\"$PROJ_NAME\",\"pattern\":\"def \"}"
|
||||
else
|
||||
# ── default mode (unchanged) ─────────────────────────────────
|
||||
# Queries every 2 seconds
|
||||
mcp_call search_graph "{\"project\":\"$PROJ_NAME\",\"name_pattern\":\".*compute.*\"}"
|
||||
mcp_call trace_path "{\"project\":\"$PROJ_NAME\",\"function_name\":\"compute\",\"direction\":\"both\"}"
|
||||
|
||||
# File mutation every 2 minutes
|
||||
if [ $((NOW - LAST_MUTATE)) -ge 120 ]; then
|
||||
echo "# mutation at cycle $CYCLE $(date)" >> "$SOAK_PROJECT/src/main.py"
|
||||
git -C "$SOAK_PROJECT" add -A 2>/dev/null
|
||||
git -C "$SOAK_PROJECT" -c user.email=test@test -c user.name=test commit -q -m "cycle $CYCLE" 2>/dev/null || true
|
||||
LAST_MUTATE=$NOW
|
||||
fi
|
||||
|
||||
# Full reindex every 2 minutes (compressed — simulates 15min real interval)
|
||||
if [ $((NOW - LAST_REINDEX)) -ge 120 ]; then
|
||||
mcp_call index_repository "{\"repo_path\":\"$SOAK_PROJECT\"}"
|
||||
LAST_REINDEX=$NOW
|
||||
fi
|
||||
fi
|
||||
|
||||
# Collect diagnostics every 10 seconds (5 cycles)
|
||||
if [ $((CYCLE % 5)) -eq 0 ]; then
|
||||
collect_snapshot
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# ── Phase 4: Idle period + final snapshot ────────────────────────
|
||||
|
||||
echo "--- Phase 4: idle (30s) ---"
|
||||
sleep 30
|
||||
collect_snapshot
|
||||
|
||||
# Check idle CPU
|
||||
IDLE_CPU=$(ps -o %cpu= -p "$SERVER_PID" 2>/dev/null | tr -d ' ' || echo "0")
|
||||
echo "OK: idle CPU=${IDLE_CPU}%"
|
||||
|
||||
# ── Phase 5: Crash recovery test ────────────────────────────────
|
||||
# Skipped in query-leak mode: crash recovery re-indexes (Phase 5 calls
|
||||
# index_repository), which triggers cbm_mem_collect and would mask the #581
|
||||
# query-only leak the whole run is trying to surface.
|
||||
|
||||
if [ "$SKIP_CRASH" != "--skip-crash-test" ] && [ "$CBM_SOAK_MODE" != "query-leak" ]; then
|
||||
echo "--- Phase 5: crash recovery ---"
|
||||
|
||||
# Kill server mid-operation, restart, verify clean index
|
||||
mcp_call index_repository "{\"repo_path\":\"$SOAK_PROJECT\"}"
|
||||
kill -9 "$SERVER_PID" 2>/dev/null || true
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
exec 3>&- 4<&-
|
||||
|
||||
# Restart server
|
||||
CBM_DIAGNOSTICS=1 "$BINARY" < "$SERVER_IN" > "$SERVER_OUT" 2>>"$RESULTS_DIR/server-stderr.log" &
|
||||
SERVER_PID=$!
|
||||
exec 3>"$SERVER_IN"
|
||||
exec 4<"$SERVER_OUT"
|
||||
sleep 3
|
||||
|
||||
if kill -0 "$SERVER_PID" 2>/dev/null; then
|
||||
echo "OK: server restarted after kill -9"
|
||||
echo '{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"capabilities":{}}}' >&3
|
||||
read -t 10 INIT_RESP <&4 || true
|
||||
|
||||
# Verify clean re-index works
|
||||
mcp_call index_repository "{\"repo_path\":\"$SOAK_PROJECT\"}"
|
||||
echo "OK: clean re-index after crash recovery"
|
||||
else
|
||||
echo "FAIL: server did not restart after kill -9"
|
||||
PASS=false
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Phase 6: Shutdown + analysis ─────────────────────────────────
|
||||
|
||||
echo "--- Phase 6: shutdown + analysis ---"
|
||||
exec 3>&- # close server stdin → EOF → clean exit
|
||||
sleep 2
|
||||
exec 4<&- # close stdout reader
|
||||
kill "$SERVER_PID" 2>/dev/null || true
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
rm -f "$SERVER_IN" "$SERVER_OUT"
|
||||
|
||||
# Final diagnostics (written by thread before exit)
|
||||
FINAL_DIAG="/tmp/cbm-diagnostics-${SERVER_PID}.json"
|
||||
|
||||
# ── Analysis ─────────────────────────────────────────────────────
|
||||
|
||||
PASS=true
|
||||
|
||||
# Check 1: Memory leak detection via RSS trend
|
||||
# This is the primary leak detector on ALL platforms (including Windows
|
||||
# where LeakSanitizer is unavailable). Catches both linear leaks (slope)
|
||||
# and step-function leaks (first vs last comparison).
|
||||
TOTAL_SAMPLES=$(awk -F, 'NR>1 && $3>0 { n++ } END { print n+0 }' "$METRICS_CSV")
|
||||
MAX_RSS=$(awk -F, 'NR>1 && $3>0 { if ($3>max) max=$3 } END { printf "%.0f", max/1024/1024 }' "$METRICS_CSV")
|
||||
FIRST_RSS=$(awk -F, 'NR==2 && $3>0 { printf "%.0f", $3/1024/1024 }' "$METRICS_CSV")
|
||||
LAST_RSS=$(awk -F, '$3>0 { last=$3 } END { printf "%.0f", last/1024/1024 }' "$METRICS_CSV")
|
||||
echo "RSS: first=${FIRST_RSS}MB last=${LAST_RSS}MB max=${MAX_RSS}MB (${TOTAL_SAMPLES} samples)" | tee -a "$SUMMARY"
|
||||
|
||||
# Absolute ceiling — catches catastrophic leaks on any run length
|
||||
if [ "${MAX_RSS:-0}" -gt 200 ] 2>/dev/null; then
|
||||
echo "FAIL: RSS ${MAX_RSS}MB > 200MB ceiling" | tee -a "$SUMMARY"
|
||||
PASS=false
|
||||
fi
|
||||
|
||||
# Slope — informational for short runs, enforced only for runs >= 30 min
|
||||
# (10-min runs have too few post-warmup samples for reliable regression)
|
||||
RSS_SLOPE=$(awk -F, -v skip="$((TOTAL_SAMPLES / 5))" '
|
||||
NR>1 && $3>0 {
|
||||
row++
|
||||
if (row <= skip) next
|
||||
n++; x=$1; y=$3; sx+=x; sy+=y; sxx+=x*x; sxy+=x*y
|
||||
}
|
||||
END {
|
||||
if (n<5) { print 0; exit }
|
||||
slope = (n*sxy - sx*sy) / (n*sxx - sx*sx)
|
||||
printf "%.0f", slope * 3600 / 1024
|
||||
}' "$METRICS_CSV")
|
||||
echo "RSS slope (post-warmup): ${RSS_SLOPE} KB/hr" | tee -a "$SUMMARY"
|
||||
if [ "$DURATION_MIN" -ge 30 ] && [ "${RSS_SLOPE:-0}" -gt 500 ] 2>/dev/null; then
|
||||
echo "FAIL: RSS slope ${RSS_SLOPE} KB/hr > 500 KB/hr" | tee -a "$SUMMARY"
|
||||
PASS=false
|
||||
fi
|
||||
|
||||
# Check 1b: RSS ratio (last / first) — catches step-function leaks
|
||||
if [ "${FIRST_RSS:-0}" -gt 0 ] 2>/dev/null; then
|
||||
RSS_RATIO=$(awk "BEGIN { printf \"%.1f\", ${LAST_RSS} / ${FIRST_RSS} }")
|
||||
echo "RSS ratio (last/first): ${RSS_RATIO}x" | tee -a "$SUMMARY"
|
||||
if awk "BEGIN { exit (${LAST_RSS} / ${FIRST_RSS} > 3.0) ? 0 : 1 }" 2>/dev/null; then
|
||||
echo "FAIL: RSS grew ${RSS_RATIO}x (last=${LAST_RSS}MB vs first=${FIRST_RSS}MB)" | tee -a "$SUMMARY"
|
||||
PASS=false
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check 2: FD drift
|
||||
FD_DRIFT=$(awk -F, 'NR>1 && $5>0 { if (!first) first=$5; last=$5 } END { print last-first }' "$METRICS_CSV")
|
||||
echo "FD drift: ${FD_DRIFT:-0}" | tee -a "$SUMMARY"
|
||||
if [ "${FD_DRIFT:-0}" -gt 20 ] 2>/dev/null; then
|
||||
echo "FAIL: FD drift ${FD_DRIFT} > 20" | tee -a "$SUMMARY"
|
||||
PASS=false
|
||||
fi
|
||||
|
||||
# Check 3: Idle CPU
|
||||
IDLE_INT=$(echo "$IDLE_CPU" | cut -d. -f1)
|
||||
echo "Idle CPU: ${IDLE_CPU}%" | tee -a "$SUMMARY"
|
||||
if [ "${IDLE_INT:-0}" -gt 5 ] 2>/dev/null; then
|
||||
echo "FAIL: idle CPU ${IDLE_CPU}% > 5%" | tee -a "$SUMMARY"
|
||||
PASS=false
|
||||
fi
|
||||
|
||||
# Check 4: Max query latency (exclude index_repository — indexing is legitimately slow)
|
||||
MAX_LATENCY=$(awk -F, 'NR>1 && $2!="index_repository" { if ($3>max) max=$3 } END { print max+0 }' "$LATENCY_CSV")
|
||||
MAX_INDEX=$(awk -F, 'NR>1 && $2=="index_repository" { if ($3>max) max=$3 } END { print max+0 }' "$LATENCY_CSV")
|
||||
echo "Max query latency: ${MAX_LATENCY}ms (index: ${MAX_INDEX}ms)" | tee -a "$SUMMARY"
|
||||
# 60s threshold — MSYS2/Wine adds significant overhead to all operations
|
||||
if [ "${MAX_LATENCY:-0}" -gt 60000 ] 2>/dev/null; then
|
||||
echo "FAIL: max query latency ${MAX_LATENCY}ms > 60s" | tee -a "$SUMMARY"
|
||||
PASS=false
|
||||
fi
|
||||
|
||||
# Check 5: Query count (sanity — should have many)
|
||||
TOTAL_QUERIES=$(awk -F, 'NR>1 { n++ } END { print n+0 }' "$LATENCY_CSV")
|
||||
echo "Total queries: $TOTAL_QUERIES" | tee -a "$SUMMARY"
|
||||
|
||||
# ── Cleanup ──────────────────────────────────────────────────────
|
||||
|
||||
rm -rf "$SOAK_PROJECT"
|
||||
|
||||
echo ""
|
||||
if $PASS; then
|
||||
echo "=== soak-test: PASSED ===" | tee -a "$SUMMARY"
|
||||
else
|
||||
echo "=== soak-test: FAILED ===" | tee -a "$SUMMARY"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,169 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Run the native-Windows product-surface test suite for codebase-memory-mcp.
|
||||
|
||||
.DESCRIPTION
|
||||
Builds the product binary (build/c/codebase-memory-mcp.exe) if it is not
|
||||
already present, then runs the deterministic Windows integration tests under
|
||||
tests/windows/ against a real codebase-memory-mcp.exe (real stdio / CLI /
|
||||
HTTP UI, real SQLite DB).
|
||||
|
||||
Two categories of test:
|
||||
|
||||
GUARDS - regression guards for Windows bugs already fixed on main.
|
||||
They must stay GREEN (exit 0); a RED (exit 1) means the fix
|
||||
regressed and fails this runner.
|
||||
* test_non_ascii_path.py guards #636/#357 (fixed by #700)
|
||||
* test_non_ascii_cache_dump.py guards #996 (writer cbm_fopen)
|
||||
* test_hook_augment.py guards #618 (fixed by #619)
|
||||
* test_ui_drive_listing.py guards #548 (roots field)
|
||||
* test_cli_non_ascii_arg.py guards #423/#20 (wide-argv main())
|
||||
|
||||
KNOWN REDS - genuine, still-open Windows bugs reproduced at the product
|
||||
surface. They are EXPECTED to be RED (exit 1) and are opt-in
|
||||
(never gate CI). If one turns GREEN the underlying bug was
|
||||
fixed and it should be promoted to a guard.
|
||||
* (none currently - test_cli_non_ascii_arg.py was promoted to a
|
||||
guard when the wide-argv fix for #423/#20 landed)
|
||||
|
||||
Determinism: the runner sets CBM_INDEX_SUPERVISOR=0 so the path / hook / drive
|
||||
guards index in-process (the pass-level readers under test, e.g. #700's cbm_fopen
|
||||
routing, run in-process either way). The non-ASCII CLI guard is the exception - it
|
||||
drops that override to cross the real supervisor -> worker spawn, where the second
|
||||
half of #423/#20 lives (CreateProcessW delivering the wide command line).
|
||||
|
||||
On native Windows the MinGW/LLVM toolchain ships no libasan/libubsan, so the
|
||||
build disables sanitizers (SANITIZE=). Where the toolchain provides
|
||||
AddressSanitizer/UBSan (Linux containers, WSL), prefer scripts/test.sh.
|
||||
|
||||
.PARAMETER Binary
|
||||
Path to an existing codebase-memory-mcp.exe. If omitted, the script builds it
|
||||
(target selected by -Target) into build/c/.
|
||||
|
||||
.PARAMETER Target
|
||||
Makefile.cbm target used when building: 'cbm-with-ui' (default; needed for the
|
||||
drive-picker guard's embedded HTTP UI) or 'cbm' (no UI - the drive guard then
|
||||
reports a precondition and is skipped).
|
||||
|
||||
.PARAMETER GuardsOnly
|
||||
Run only the green guards (the CI gate). Skips the opt-in known-red repros.
|
||||
|
||||
.PARAMETER Make
|
||||
Path to GNU make (default: 'make' on PATH; MSYS2 ships it at
|
||||
C:\msys64\usr\bin\make.exe).
|
||||
|
||||
.EXAMPLE
|
||||
pwsh -File scripts/test-windows.ps1
|
||||
.EXAMPLE
|
||||
pwsh -File scripts/test-windows.ps1 -GuardsOnly -Binary build\c\codebase-memory-mcp.exe
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$Binary,
|
||||
[ValidateSet("cbm-with-ui", "cbm")]
|
||||
[string]$Target = "cbm-with-ui",
|
||||
[switch]$GuardsOnly,
|
||||
[string]$Make = "make"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$repoRoot = Split-Path -Parent $PSScriptRoot
|
||||
Set-Location $repoRoot
|
||||
|
||||
$python = (Get-Command python -ErrorAction SilentlyContinue)
|
||||
if (-not $python) { $python = (Get-Command py -ErrorAction SilentlyContinue) }
|
||||
if (-not $python) { throw "Python 3 is required to run the Windows tests." }
|
||||
$py = $python.Source
|
||||
|
||||
# A writable Windows temp dir that GNU make forwards to the native gcc. MSYS2
|
||||
# strips TMP/TEMP from the environment it hands native children, so pass them as
|
||||
# make command-line variables (make exports those to recipe processes).
|
||||
$tmp = $env:TEMP
|
||||
if (-not $tmp) { $tmp = "$env:USERPROFILE\AppData\Local\Temp" }
|
||||
|
||||
function Resolve-Binary {
|
||||
param([string]$Explicit)
|
||||
if ($Explicit) { return (Resolve-Path $Explicit).Path }
|
||||
$built = Join-Path $repoRoot "build\c\codebase-memory-mcp.exe"
|
||||
if (Test-Path $built) { return $built }
|
||||
Write-Host "Building $Target via Makefile.cbm ..." -ForegroundColor Cyan
|
||||
& $Make "-j" "-f" "Makefile.cbm" $Target "SANITIZE=" "TMP=$tmp" "TEMP=$tmp" "TMPDIR=$tmp"
|
||||
if ($LASTEXITCODE -ne 0) { throw "build failed (exit $LASTEXITCODE)" }
|
||||
if (-not (Test-Path $built)) { throw "binary not produced at $built" }
|
||||
return $built
|
||||
}
|
||||
|
||||
$bin = Resolve-Binary -Explicit $Binary
|
||||
Write-Host "Binary: $bin" -ForegroundColor Green
|
||||
|
||||
$env:PYTHONUTF8 = "1" # encode argv/stdio as UTF-8
|
||||
$env:CBM_INDEX_SUPERVISOR = "0" # in-process indexing (see .DESCRIPTION)
|
||||
|
||||
# Green regression guards - must stay GREEN (exit 0). RED (exit 1) = the fix for
|
||||
# the referenced issue regressed. The drive-picker guard needs the embedded HTTP
|
||||
# UI (build target cbm-with-ui); against a non-UI binary it reports a precondition
|
||||
# (exit 2) and is skipped rather than failed.
|
||||
$guards = @(
|
||||
"tests\windows\test_non_ascii_path.py",
|
||||
"tests\windows\test_non_ascii_cache_dump.py",
|
||||
"tests\windows\test_hook_augment.py",
|
||||
"tests\windows\test_ui_drive_listing.py",
|
||||
"tests\windows\test_cli_non_ascii_arg.py"
|
||||
)
|
||||
|
||||
# Opt-in known-red repros - EXPECTED red (exit 1); never gate CI. Currently empty:
|
||||
# test_cli_non_ascii_arg.py was promoted to a guard when #423/#20's wide-argv fix landed.
|
||||
$knownReds = @()
|
||||
|
||||
$guardFailures = @()
|
||||
$guardSkips = @()
|
||||
$fixedKeepers = @()
|
||||
|
||||
Write-Host "`n--- Green guards ---" -ForegroundColor Cyan
|
||||
foreach ($t in $guards) {
|
||||
Write-Host "`n=== $t ===" -ForegroundColor Cyan
|
||||
& $py $t $bin
|
||||
$code = $LASTEXITCODE
|
||||
if ($code -eq 0) {
|
||||
Write-Host "GREEN ($t)" -ForegroundColor Green
|
||||
} elseif ($code -eq 1) {
|
||||
Write-Host "RED ($t) - REGRESSION: a fixed Windows bug is broken again" -ForegroundColor Red
|
||||
$guardFailures += $t
|
||||
} else {
|
||||
Write-Host "PRECONDITION ($t) exit=$code - skipped (see message above)" -ForegroundColor Yellow
|
||||
$guardSkips += $t
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $GuardsOnly) {
|
||||
Write-Host "`n--- Known reds (opt-in, expected red) ---" -ForegroundColor Cyan
|
||||
foreach ($t in $knownReds) {
|
||||
Write-Host "`n=== $t ===" -ForegroundColor Cyan
|
||||
& $py $t $bin
|
||||
$code = $LASTEXITCODE
|
||||
if ($code -eq 1) {
|
||||
Write-Host "RED ($t) - expected; the underlying Windows bug is still open" -ForegroundColor DarkYellow
|
||||
} elseif ($code -eq 0) {
|
||||
Write-Host "GREEN ($t) - the bug appears FIXED; promote this to a guard" -ForegroundColor Green
|
||||
$fixedKeepers += $t
|
||||
} else {
|
||||
Write-Host "PRECONDITION ($t) exit=$code - skipped (see message above)" -ForegroundColor Yellow
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
if ($guardSkips.Count -gt 0) {
|
||||
Write-Host ("Guards skipped (precondition): {0} - e.g. the drive-picker guard " -f $guardSkips.Count) -ForegroundColor Yellow
|
||||
Write-Host "needs a UI build (-Target cbm-with-ui, the default)." -ForegroundColor Yellow
|
||||
}
|
||||
if ($fixedKeepers.Count -gt 0) {
|
||||
Write-Host ("Known-red repros that are now GREEN (promote to guards): {0}" -f ($fixedKeepers -join ", ")) -ForegroundColor Green
|
||||
}
|
||||
if ($guardFailures.Count -gt 0) {
|
||||
Write-Host ("REGRESSION: {0} green guard(s) went red: {1}" -f $guardFailures.Count, ($guardFailures -join ", ")) -ForegroundColor Red
|
||||
Write-Host "A previously-fixed Windows bug is broken again (see the guard's docstring and its referenced issue)." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
Write-Host "All Windows green guards passed." -ForegroundColor Green
|
||||
exit 0
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
# test.sh — Clean build + run all C tests with ASan + UBSan.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/test.sh # Auto-detect everything
|
||||
# scripts/test.sh --arch x86_64 # Force x86_64 build
|
||||
# scripts/test.sh CC=gcc-14 CXX=g++-14 # Override compiler
|
||||
#
|
||||
# This script is the SINGLE source of truth for running tests.
|
||||
# Used identically in local development and CI workflows.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Parse --arch flag before sourcing env.sh
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--arch) :;; # next arg is the value, handled below
|
||||
arm64|x86_64)
|
||||
# Check if previous arg was --arch
|
||||
if [[ "${prev_arg:-}" == "--arch" ]]; then
|
||||
export CBM_ARCH="$arg"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
prev_arg="$arg"
|
||||
done
|
||||
|
||||
# Also support --arch=value
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--arch=*) export CBM_ARCH="${arg#--arch=}" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# shellcheck source=env.sh
|
||||
source "$ROOT/scripts/env.sh"
|
||||
|
||||
# Forward CC/CXX and collect make-passthrough args
|
||||
MAKE_ARGS=""
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
CC=*|CXX=*) export "${arg}" ;;
|
||||
--arch|--arch=*) ;; # already handled
|
||||
arm64|x86_64) ;; # already handled
|
||||
*=*) MAKE_ARGS="$MAKE_ARGS $arg" ;; # forward any VAR=VAL to make
|
||||
esac
|
||||
done
|
||||
|
||||
print_env "test.sh"
|
||||
|
||||
# Verify compiler supports target arch
|
||||
verify_compiler "$CC"
|
||||
|
||||
# Step 1: Clean
|
||||
scripts/clean.sh
|
||||
|
||||
# Step 2 + 3: Build and run tests (Makefile applies $ARCHFLAGS on macOS)
|
||||
make -j"$NPROC" -f Makefile.cbm test $MAKE_ARGS
|
||||
|
||||
# Step 4: C++ large-TU index-hang regression guard (#410). Runs the PROD binary
|
||||
# in a subprocess with a wall-clock timeout — a hang must fail, not block the run.
|
||||
# Opt-in via CBM_RUN_HANG_TEST=1 (it needs the prod binary, which the ASan unit
|
||||
# run above does not build). Skipped by default so the fast unit run stays fast.
|
||||
if [ "${CBM_RUN_HANG_TEST:-0}" = "1" ]; then
|
||||
echo "=== Step 4: C++ index-hang regression (#410) ==="
|
||||
bash "$ROOT/tests/test_cpp_index_hang.sh"
|
||||
fi
|
||||
|
||||
# Step 5: Parent-death watchdog regression (#406/#407). Builds the prod stdio
|
||||
# binary and verifies it self-exits when its launching parent is killed.
|
||||
echo "=== Step 5: parent-death watchdog regression (#406/#407) ==="
|
||||
make -j"$NPROC" -f Makefile.cbm cbm $MAKE_ARGS
|
||||
bash "$ROOT/tests/test_parent_watchdog.sh"
|
||||
|
||||
# Step 5b: worker-mode parent-death watchdog (#845). A supervised index worker
|
||||
# (`cli --index-worker …`) whose supervisor dies must self-exit instead of
|
||||
# indexing on as an orphan. Reuses the prod binary built in Step 5.
|
||||
echo "=== Step 5b: worker-mode watchdog regression (#845) ==="
|
||||
bash "$ROOT/tests/test_worker_watchdog.sh"
|
||||
|
||||
# Step 6: security-strings URL allow-list regression. The MSYS2 CLANG64 toolchain
|
||||
# bakes its package-tracker URL into the static Windows .exe; the binary string
|
||||
# audit must allow-list it (Windows-only — Linux smoke never saw it).
|
||||
echo "=== Step 6: security-strings allow-list regression ==="
|
||||
bash "$ROOT/tests/test_security_strings_allowlist.sh"
|
||||
|
||||
echo "=== All tests passed ==="
|
||||
Executable
+98
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Integration test for the poll/getline FILE* buffering fix.
|
||||
|
||||
Spawns the MCP server binary, sends initialize + notifications/initialized +
|
||||
tools/list all at once (no delays), and asserts that the tools/list response
|
||||
arrives within 5 seconds.
|
||||
|
||||
Usage:
|
||||
python3 scripts/test_mcp_rapid_init.py [/path/to/binary]
|
||||
|
||||
Exit codes:
|
||||
0 - PASS
|
||||
1 - FAIL
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
|
||||
TIMEOUT_S = 5
|
||||
|
||||
MESSAGES = (
|
||||
b'{"jsonrpc":"2.0","id":1,"method":"initialize",'
|
||||
b'"params":{"protocolVersion":"2025-11-25","capabilities":{}}}\n'
|
||||
b'{"jsonrpc":"2.0","method":"notifications/initialized"}\n'
|
||||
b'{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}\n'
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) >= 2:
|
||||
binary = sys.argv[1]
|
||||
else:
|
||||
# Default: look for build artifact relative to this script's directory
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
repo_root = os.path.dirname(script_dir)
|
||||
binary = os.path.join(repo_root, "build", "c", "codebase-memory-mcp")
|
||||
|
||||
if not os.path.isfile(binary):
|
||||
print(f"FAIL: binary not found at {binary}")
|
||||
sys.exit(1)
|
||||
|
||||
if not os.access(binary, os.X_OK):
|
||||
print(f"FAIL: binary not executable: {binary}")
|
||||
sys.exit(1)
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[binary],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
try:
|
||||
# Write all 3 messages in one call and close stdin to signal EOF
|
||||
stdout_data, _ = proc.communicate(input=MESSAGES, timeout=TIMEOUT_S)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
print(
|
||||
f"FAIL: server did not respond within {TIMEOUT_S}s "
|
||||
f"(poll/getline buffering bug not fixed)"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
output = stdout_data.decode("utf-8", errors="replace")
|
||||
|
||||
# Expect exactly 2 JSON responses: id:1 (initialize) and id:2 (tools/list).
|
||||
# notifications/initialized has no id and produces no response.
|
||||
lines = [ln.strip() for ln in output.splitlines() if ln.strip()]
|
||||
import json as _json
|
||||
json_lines = []
|
||||
for ln in lines:
|
||||
try:
|
||||
json_lines.append(_json.loads(ln))
|
||||
except _json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
ids = {obj.get("id") for obj in json_lines if "id" in obj}
|
||||
if 1 not in ids:
|
||||
print("FAIL: missing initialize response (id:1) in server output")
|
||||
print(f"Server output was:\n{output!r}")
|
||||
sys.exit(1)
|
||||
if 2 not in ids:
|
||||
print("FAIL: missing tools/list response (id:2) in server output")
|
||||
print(f"Server output was:\n{output!r}")
|
||||
sys.exit(1)
|
||||
if "tools" not in output:
|
||||
print("FAIL: tools/list response body missing 'tools' key")
|
||||
print(f"Server output was:\n{output!r}")
|
||||
sys.exit(1)
|
||||
|
||||
print("PASS")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
# vendor-grammar.sh: Vendor a single tree-sitter grammar into internal/cbm/vendored/grammars/<name>/
|
||||
# Usage: ./scripts/vendor-grammar.sh <repo_url> <name> [subdir]
|
||||
# repo_url: GitHub repository URL (e.g., https://github.com/tree-sitter/tree-sitter-json)
|
||||
# name: Target directory name (e.g., json)
|
||||
# subdir: Optional subdirectory within repo containing src/ (e.g., "fsharp" for monorepo grammars)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_URL="$1"
|
||||
NAME="$2"
|
||||
SUBDIR="${3:-}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
GRAMMAR_DIR="$PROJECT_DIR/internal/cbm/vendored/grammars/$NAME"
|
||||
TMPDIR="$(mktemp -d)"
|
||||
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
|
||||
echo "Vendoring $NAME from $REPO_URL..."
|
||||
|
||||
git clone --depth 1 "$REPO_URL" "$TMPDIR/repo" 2>/dev/null
|
||||
|
||||
SRC_DIR="$TMPDIR/repo/src"
|
||||
if [ -n "$SUBDIR" ]; then
|
||||
SRC_DIR="$TMPDIR/repo/$SUBDIR/src"
|
||||
fi
|
||||
|
||||
if [ ! -f "$SRC_DIR/parser.c" ]; then
|
||||
echo "ERROR: $SRC_DIR/parser.c not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$GRAMMAR_DIR/tree_sitter"
|
||||
|
||||
cp "$SRC_DIR/parser.c" "$GRAMMAR_DIR/"
|
||||
|
||||
if [ -f "$SRC_DIR/scanner.c" ]; then
|
||||
cp "$SRC_DIR/scanner.c" "$GRAMMAR_DIR/"
|
||||
fi
|
||||
if [ -f "$SRC_DIR/scanner.cc" ]; then
|
||||
echo "WARNING: $NAME has C++ scanner (scanner.cc) — needs special handling" >&2
|
||||
fi
|
||||
|
||||
# Copy tree_sitter headers
|
||||
if [ -d "$SRC_DIR/tree_sitter" ]; then
|
||||
cp "$SRC_DIR/tree_sitter/"*.h "$GRAMMAR_DIR/tree_sitter/" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Copy any extra headers (.h, .inc files) used by scanners
|
||||
# Examples: tag.h (Vue/Svelte/Astro), unicode.h (PureScript/Typst),
|
||||
# TokenTree.h/.inc (VHDL)
|
||||
for f in "$SRC_DIR"/*.h "$SRC_DIR"/*.inc; do
|
||||
[ -f "$f" ] && cp "$f" "$GRAMMAR_DIR/"
|
||||
done
|
||||
# Copy common/ subdirectory if present (e.g., F# scanner uses common/scanner.h)
|
||||
if [ -d "$SRC_DIR/common" ]; then
|
||||
cp -r "$SRC_DIR/common" "$GRAMMAR_DIR/"
|
||||
fi
|
||||
|
||||
# Copy LICENSE file from upstream repo
|
||||
REPO_ROOT="$TMPDIR/repo"
|
||||
if [ -n "$SUBDIR" ]; then
|
||||
# For monorepos, check subdir first, then repo root
|
||||
if [ -f "$REPO_ROOT/$SUBDIR/LICENSE" ]; then
|
||||
cp "$REPO_ROOT/$SUBDIR/LICENSE" "$GRAMMAR_DIR/LICENSE"
|
||||
elif [ -f "$REPO_ROOT/LICENSE" ]; then
|
||||
cp "$REPO_ROOT/LICENSE" "$GRAMMAR_DIR/LICENSE"
|
||||
elif [ -f "$REPO_ROOT/LICENSE.md" ]; then
|
||||
cp "$REPO_ROOT/LICENSE.md" "$GRAMMAR_DIR/LICENSE"
|
||||
elif [ -f "$REPO_ROOT/COPYING" ]; then
|
||||
cp "$REPO_ROOT/COPYING" "$GRAMMAR_DIR/LICENSE"
|
||||
else
|
||||
echo "WARNING: No LICENSE file found for $NAME" >&2
|
||||
fi
|
||||
else
|
||||
if [ -f "$REPO_ROOT/LICENSE" ]; then
|
||||
cp "$REPO_ROOT/LICENSE" "$GRAMMAR_DIR/LICENSE"
|
||||
elif [ -f "$REPO_ROOT/LICENSE.md" ]; then
|
||||
cp "$REPO_ROOT/LICENSE.md" "$GRAMMAR_DIR/LICENSE"
|
||||
elif [ -f "$REPO_ROOT/COPYING" ]; then
|
||||
cp "$REPO_ROOT/COPYING" "$GRAMMAR_DIR/LICENSE"
|
||||
else
|
||||
echo "WARNING: No LICENSE file found for $NAME" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Vendored $NAME to $GRAMMAR_DIR"
|
||||
ls -la "$GRAMMAR_DIR/"
|
||||
@@ -0,0 +1,75 @@
|
||||
dd4f25cae53209d45d73f8e6a2b9c219e8fc7434d97f20eba9af1a8b850030fd /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/include/mimalloc-new-delete.h
|
||||
21fcf61c4443341ac6bf6ea528af31dc7267e8e3456fc64bfd07704503032175 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/include/mimalloc-override.h
|
||||
5260df227ea1612b8678a06b063cc8f48e739b551554541cdeb4d430a7e257b3 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/include/mimalloc-stats.h
|
||||
869161433ab2d807a75752d0762accf5b68b6b24ce67e417af46bf181b9b6c72 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/include/mimalloc.h
|
||||
e9512b521a82cc0cbc262f6060dd70bb4243ac4a7d69d6ab8b1ef8e56f181a21 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/include/mimalloc/atomic.h
|
||||
57a7297e18ae464a7bfd3fc25a46229008602385697305c69ed289b89ece7631 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/include/mimalloc/bits.h
|
||||
5aeeff21d667daf4aadcc66f634aa957ecd688d15b277828f367c8c2274251a8 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/include/mimalloc/internal.h
|
||||
555302d2fcd05e974441d0c9c9208c26e860db4b3f17b9025a08f42e0f4807ba /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/include/mimalloc/prim.h
|
||||
1f378b4b6063c078d2aeb5d916c1d30b5db787a44ab9118e7daf673b086882c6 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/include/mimalloc/track.h
|
||||
b54e760da2a8d9d8a9cb708e280268c2d194c99bc518b21f72db70100578ad6a /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/include/mimalloc/types.h
|
||||
70420b4430da4bb6b5197fd27c6c0df485249c53a986bc9536c9fb643fc7613b /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/alloc-aligned.c
|
||||
6c8e9b0d042481c55791edb7330daec64bedaf26f0e70c9ca40de3cc17a690fe /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/alloc-override.c
|
||||
98d0eb877d7b9cdf7e9f857aaf3c739b69f746b91221e0e427e82bdd7cec90b3 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/alloc-posix.c
|
||||
a92d58e47220e5b6d138b83d92498771be4195ee59136a134e4461f6f726b9b8 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/alloc.c
|
||||
86fde644ede2e8d582e53499568a1713516a20326e0350eaa06a28029cece91b /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/arena-meta.c
|
||||
a1d6900d2b9a17461aa2a72a7791dfb22f6167b86f7ac6f15f981d9735aa2adb /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/arena.c
|
||||
de16033e029488e9ade4152ea1274e1a5ecbdc2c8a8d8ee50b745fd037ee6f6c /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/bitmap.c
|
||||
2e7a54ff44592a1fe8fe929e691322622f8aa1e0b1e80954781f332a88dd6cd1 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/bitmap.h
|
||||
c9a845dd7b9020f26d73761d235fb75e7bc4a3682576de750050ac4098211b43 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/free.c
|
||||
ae8e88737c9861e8e071fe5f1e7acc52093bcffd136a359a6b473101e968c14f /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/heap.c
|
||||
eca1fe77d72ab2aaccc447c8119056e9c4e8851bda9a95472b573539f76107ad /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/init.c
|
||||
ebd95b475f15d5387e6ef3c88a9cb47b88e64d7e55998ae1dc656614676a443b /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/libc.c
|
||||
96ef01e41f4b3043118e8c801e660b1d3f185d6fb824e5298a038aee89b6a392 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/options.c
|
||||
011e965f28c0e2c885a6834d51bd756a458dd37ebfed6fd4d9ad6607ce0390ed /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/os.c
|
||||
1be3a208c678fc34187c2a8ffb75f3fe1eecc674bbf8b2af4d1f329506b1fad5 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/page-map.c
|
||||
7ec275805c9f00eb66710ffa99817b3ed1dba9826435d08f4866f7935152f6ae /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/page-queue.c
|
||||
d3614a59f2407a32e75b051deb1aff31b233f587d325dcc5420ba508551b687e /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/page.c
|
||||
7488e9e562ca85d2d757ec48e4d334935263a2333aa05b6061d8520f427c5890 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/prim/emscripten/prim.c
|
||||
ce14eb6dfd55f241095b55f70983931a2d92f7a6a622a9b360cdd70ab3b53208 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/prim/osx/alloc-override-zone.c
|
||||
247a9952465eb105be03a9962e922b085ce5d52034775551a61cd8594275be73 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/prim/osx/prim.c
|
||||
cc771788e5ba591efb681829a8724f8662a1f50f3db39cf5ebe7738b8ed57e44 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/prim/prim.c
|
||||
c1ae32eaac7ccf18c10058cca99723892f8bd027318590151f5011c929c7e3be /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/prim/unix/prim.c
|
||||
bee9c655a17e1e93bf4908e0a902fd3b566628447b1446fcbfa92bca2e76ca15 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/prim/wasi/prim.c
|
||||
4f3110ef2054c95cb275be96cb279224d3d46a728c5117bd1435425f382de778 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/prim/windows/etw.h
|
||||
5cde60fcef94cd79ab816030be00f8c1bdf18fab343563508c681ec6e762d1f1 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/prim/windows/prim.c
|
||||
a96a1acc75fd6595811b28420cf0495b87fac98c4dbc73aa0d742b6824f607c8 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/random.c
|
||||
4059be2a6676693be3433f5aedc711f56301772e65f3573654c236d6d1cfd074 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/static.c
|
||||
16825bbc252de422982b5728df7eda181d952824f5e3ce958c7fc85feae64b8e /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/stats.c
|
||||
7f76bcace4b0578f2063477752138adabd99f7b10a276998e296a312c47a7cfe /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/theap.c
|
||||
49be69a4b674e09690a63870830f3ffb05698ed4210949ef5add11957a08325b /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/threadlocal.c
|
||||
dab3009c0d76b0c5e05ad8abc7c9f8f6effca547fea3c0394be96418a85c1081 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/nomic/code_tokens.h
|
||||
494d329d06e33904e6264b1f9d1cc82de2c6bb212f8d78ba281b7e1eb1179b61 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/nomic/code_vectors.h
|
||||
9512509b1bccb7461f79bea8aad6280ae4699e925fa4804381b71f59e7efb0c5 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/sqlite3/sqlite3.c
|
||||
19585c8b5230e9d4f223bf31b709ece7b6a0bb3faf00d8310625d8e58cda1b1d /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/sqlite3/sqlite3.h
|
||||
ea81fb7bd05882e0e0b92c4d60f677b205f7f1fbf085f218b12f0b5b3f0b9e48 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/sqlite3/sqlite3ext.h
|
||||
7efd127c0fc4fe26a07684345cce9287762346abeab665e2fe72711c6fc118bd /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/regcomp.c
|
||||
eab23b8e79ee90f78e8495de64519afb61b627e062804fd4a622784e052a85fa /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/regerror.c
|
||||
26b0f550d491335cdaa3fecfe49213d68466befdf648ed281ccdaa631ea6d4f9 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/regex.h
|
||||
fd6fe2789439d3d28140c27edfe6bcdde1d1c737cab4bd27b1287d3e759fa82d /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/regexec.c
|
||||
90f76dce41eade7e28c3477d8b45acb8d2ccbc6d4aaba0bb93f0d5ec5b160820 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre_all.c
|
||||
e98c7732fdbb35ec182edfe043743d7e6b4ad7bcf57b815ec9f37f0d1065a062 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-ast.c
|
||||
f5d0374597a42f4bf0e7a80001a68bae9ea2622b80f760d8005200fd20acaf0f /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-ast.h
|
||||
45407a83ef0151a977cb7f8a5275b2a9831ae570d6f27a43723c8da1e76c0261 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-compile.c
|
||||
924c8b9aa6a261d8b86f1b0b3b575adc5274e135fefcd4193497445e5fc6245a /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-compile.h
|
||||
6d803fb5dd3cdb8af353936869d92f7b6e25644c7fb3a26280d24fc4451db7d2 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-config.h
|
||||
d463e509cdc7eda5154d29c75791ccd58eff5d6f4345344829517f62ddb606e1 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-filter.c
|
||||
aabcc5902193f76e457deffa1b3cf2c3b63d39489e6da5fc3f16bb26e83dc067 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-filter.h
|
||||
fd6121af43c4d64f3dc6b55dcd0618bd5b69d5ada60f951bc663a924374a082c /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-internal.h
|
||||
2fcb1bbadc845bc32b73b2882bb7f1988f7fdb183b8349750e5f20b29b6223da /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-match-approx.c
|
||||
80a0b950fc1c34773d49fcec0f78a3b7de7c893852dcfc83f7eece3ebe262afa /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-match-backtrack.c
|
||||
446bf71b5c22ee432dc42705c63abcda8eaebeafecc811d682b658b8d18e68f1 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-match-parallel.c
|
||||
3f726919232c0311daa533fbaff6bdcc3746817b48672c6ed95a6e160c2877f1 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-match-utils.h
|
||||
1645537ca85eefe543da3187b3d1a65bf796ae0a0576545457b56fb63fc13c67 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-mem.c
|
||||
6ae203e4ff329bbc15bd48004f0b5ac0804fe95b1557883d6840949e75f2018b /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-mem.h
|
||||
a04e1bea47aff5d858460c1d08aac6ed3a3c8ee285500281dd3147ff0621095e /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-parse.c
|
||||
29d69be4d03e723e4b99e9887774970f7e62176aed3369faacd34b955ffb509f /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-parse.h
|
||||
4c9af903178f5f7030962b5708d4e656f8b060e795852e1eee883696b682849d /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-stack.c
|
||||
aabe11f1b7c6c627dc9cfb62cfb9565ac9ebdf2c51c2d55c5320af5db76c5e3b /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre-stack.h
|
||||
1c2d81474d2b59b7a39f5b1592473adfb4109fad99156588088f2ccc56c654ab /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/tre.h
|
||||
9632e5eeb20e3d328f8def0efb2e8230f5b5cb7d9f2e5680ad89caf065f8b3a6 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/xmalloc.c
|
||||
22aee25e6892e97719ec4a5fad91345cd2145722ebbd3ec31397aff68108e3e6 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/tre/xmalloc.h
|
||||
5c3591fe6e6c86a619eb26760e9520e37a6fd5152882ab5ad93f912e2a855966 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/xxhash/xxhash.c
|
||||
86d0d813745821bbccf0be6d67356846f138e5c20164c52c24fade1419afdf7d /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/xxhash/xxhash.h
|
||||
1da0205abb1a27c27db397b7f8a475abc0af58963af9a88fc807a6c4fa7a8d51 /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/yyjson/yyjson.c
|
||||
b2bd3aec324a0d6bc67196f647c966870e783c2e3944684f10db26b2c04b773f /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/yyjson/yyjson.h
|
||||
Reference in New Issue
Block a user