chore: import upstream snapshot with attribution
CI / Test (macos-latest) (push) Waiting to run
CI / Test (windows-latest) (push) Waiting to run
CI / Build (no embeddings / no ORT) (push) Waiting to run
CI / Format (push) Waiting to run
CI / Cookbook (Node) (push) Waiting to run
CI / Pi Extension (Node) (push) Waiting to run
CI / Rust SDK (lean-ctx-client) (push) Waiting to run
CI / Embed SDK (lean-ctx-sdk) (push) Waiting to run
CI / Python SDK (leanctx) (push) Waiting to run
CI / Hermes Plugin (Python) (push) Waiting to run
CI / SDK Conformance Matrix (push) Waiting to run
CI / Coverage (push) Waiting to run
CI / cargo-deny (push) Waiting to run
CI / Adversarial Safety (push) Waiting to run
CodeQL / Analyze (javascript-typescript) (push) Waiting to run
JetBrains Plugin / Actionlint (push) Waiting to run
CI / Benchmarks (push) Waiting to run
CI / Output-Quality Gate (eval A/B) (push) Waiting to run
CI / Documentation (push) Waiting to run
CI / CI Green (push) Blocked by required conditions
CodeQL / Analyze (actions) (push) Waiting to run
CodeQL / Analyze (rust) (push) Waiting to run
JetBrains Plugin / Validation (push) Waiting to run
JetBrains Plugin / Build (push) Waiting to run
JetBrains Plugin / Test (push) Blocked by required conditions
Security Check / Security Scan (push) Waiting to run
CI / Test (ubuntu-latest) (push) Has started running
CI / Clippy (push) Has started running

This commit is contained in:
wehub-resource-sync
2026-07-13 12:35:30 +08:00
commit 26382a7ac6
2353 changed files with 629146 additions and 0 deletions
+38
View File
@@ -0,0 +1,38 @@
{
"name": "lean-ctx",
"version": "3.3.6",
"description": "Context Runtime for AI Agents — compress LLM context by up to 99%. 72 MCP tools, 10 read modes, 95+ shell patterns.",
"homepage": "https://leanctx.com",
"repository": "https://github.com/yvgude/lean-ctx",
"license": "Apache-2.0",
"install": {
"command": "curl -fsSL https://leanctx.com/install.sh | sh",
"alternatives": {
"cargo": "cargo install lean-ctx",
"brew": "brew tap yvgude/lean-ctx && brew install lean-ctx",
"npm": "npm install -g lean-ctx-bin"
}
},
"mcp": {
"command": "lean-ctx",
"args": ["--mcp"],
"env": {}
},
"skills": ["skills/lean-ctx"],
"capabilities": [
"file-read",
"file-edit",
"shell-exec",
"code-search",
"semantic-search",
"session-management",
"knowledge-base",
"multi-agent",
"code-graph"
],
"languages": [
"rust", "typescript", "javascript", "python", "go", "java",
"c", "cpp", "csharp", "ruby", "php", "swift", "kotlin",
"scala", "hcl", "yaml", "toml", "elixir"
]
}
+4
View File
@@ -0,0 +1,4 @@
# Keep the gateway image build context lean: only rust/ is copied
# (docker/Dockerfile.gateway), and build artifacts must never leak in.
rust/target/
**/.DS_Store
+18
View File
@@ -0,0 +1,18 @@
# Code-generated reference docs are emitted with LF by `gen_docs`. Pin them to
# LF on every platform so the docs-drift gate (`cargo test` + `gen_docs --check`)
# is byte-stable — Windows checkouts would otherwise convert them to CRLF and
# fail the comparison.
docs/reference/generated/*.md text eol=lf
# Testbench fixtures, suites, recordings and lock feed `RecordedRunner` replay
# keys verbatim (file bytes are embedded into the assembled context that is
# hashed). Pin them to LF so Windows checkouts don't convert them to CRLF and
# break the byte-identical key match (off-vs-on testbench, #611/#498).
rust/eval/testbench/** text eol=lf
# Registry snapshots are generated by `gen_registry` with LF and compared
# byte-for-byte in CI and lib tests (GH #726). Same rationale as the
# generated docs above.
rust/data/addon_registry.json text eol=lf
rust/data/grammar_registry.json text eol=lf
+32
View File
@@ -0,0 +1,32 @@
#!/bin/bash
set -euo pipefail
cd "$(git rev-parse --show-toplevel)/rust"
if ! cargo fmt --check 2>/dev/null; then
echo "pre-commit: formatting check failed. Run 'cargo fmt' first."
exit 1
fi
if ! cargo clippy --all-targets -- -D warnings 2>/dev/null; then
echo "pre-commit: clippy check failed. Fix warnings before committing."
exit 1
fi
# Generated reference docs must match the tool/config definitions —
# the CI Documentation job runs the same check and fails otherwise.
if git diff --cached --name-only | grep -qE '^rust/src/(tools|tool_defs|core/config)'; then
if ! cargo run --quiet --example gen_docs --features dev-tools -- --check 2>/dev/null; then
echo "pre-commit: generated docs out of date. Run: cd rust && cargo run --example gen_docs --features dev-tools"
exit 1
fi
fi
# The committed testbench recording must match the fixtures + assembly/judge framing —
# the CI Documentation job runs the same check (#611).
if git diff --cached --name-only | grep -qE '^rust/(eval/testbench/|src/core/eval_ab/)'; then
if ! cargo run --quiet --example gen_testbench_recording --features dev-tools -- --check 2>/dev/null; then
echo "pre-commit: testbench recording out of date. Run: cd rust && cargo run --example gen_testbench_recording --features dev-tools"
exit 1
fi
fi
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env bash
#
# pre-push hook: blocks proprietary files from being pushed to GitHub.
# Reads blocked paths from .github-ignore (one per line).
# Pushes to any non-GitHub remote pass through unchecked.
#
# Install: git config core.hooksPath .githooks
set -euo pipefail
REMOTE_NAME="$1"
REMOTE_URL="$2"
REPO_ROOT="$(git rev-parse --show-toplevel)"
# ── Local CI-parity gate (runs for every remote) ──────────────────────
# Blocks pushes that would fail the deterministic CI jobs (fmt / clippy /
# doc / gen_docs / cross-platform compile) before they burn a full ~50-min
# CI matrix. Deduped per HEAD so pushing to multiple remotes (github +
# origin) only runs it once.
# Bypass: SKIP_PREFLIGHT=1 git push … (or git push --no-verify)
if [[ "${SKIP_PREFLIGHT:-0}" != "1" && -x "$REPO_ROOT/scripts/preflight.sh" ]]; then
HEAD_SHA="$(git rev-parse HEAD 2>/dev/null || true)"
MARKER="$(git rev-parse --git-dir)/preflight-passed"
if [[ -n "$HEAD_SHA" && "$(cat "$MARKER" 2>/dev/null || true)" == "$HEAD_SHA" ]]; then
echo "preflight: already green for ${HEAD_SHA:0:12} — skipping"
else
echo "preflight: running fast CI-parity gate (SKIP_PREFLIGHT=1 to bypass)…"
if "$REPO_ROOT/scripts/preflight.sh" fast; then
[[ -n "$HEAD_SHA" ]] && echo "$HEAD_SHA" > "$MARKER"
else
echo ""
echo "pre-push BLOCKED: preflight failed — fix above, or: SKIP_PREFLIGHT=1 git push"
exit 1
fi
fi
fi
# ── Proprietary-file guard (GitHub only) ──────────────────────────────
# Only guard pushes to GitHub
if [[ "$REMOTE_URL" != *"github.com"* && "$REMOTE_NAME" != "github" ]]; then
exit 0
fi
IGNORE_FILE="$REPO_ROOT/.github-ignore"
if [[ ! -f "$IGNORE_FILE" ]]; then
exit 0
fi
# Read blocked patterns (skip comments and blank lines)
BLOCKED=()
while IFS= read -r line; do
line="${line%%#*}" # strip inline comments
line="${line// /}" # strip spaces
[[ -z "$line" ]] && continue
BLOCKED+=("$line")
done < "$IGNORE_FILE"
if [[ ${#BLOCKED[@]} -eq 0 ]]; then
exit 0
fi
VIOLATIONS=()
while read -r LOCAL_REF LOCAL_SHA REMOTE_REF REMOTE_SHA; do
[[ -z "$LOCAL_SHA" ]] && continue
# skip delete pushes
if [[ "$LOCAL_SHA" == "0000000000000000000000000000000000000000" ]]; then
continue
fi
if [[ "$REMOTE_SHA" == "0000000000000000000000000000000000000000" ]]; then
# New branch — check all commits
RANGE="$LOCAL_SHA"
else
RANGE="$REMOTE_SHA..$LOCAL_SHA"
fi
# Get files ADDED or MODIFIED in the push range (not deletions)
FILES=$(git diff --diff-filter=ACMR --name-only "$RANGE" 2>/dev/null || git diff-tree --no-commit-id --diff-filter=ACMR --name-only -r "$LOCAL_SHA" 2>/dev/null || true)
for file in $FILES; do
for pattern in "${BLOCKED[@]}"; do
# Directory pattern (trailing slash): match prefix
if [[ "$pattern" == */ ]]; then
if [[ "$file" == "${pattern}"* ]]; then
VIOLATIONS+=(" $file (blocked by: $pattern)")
fi
else
# Exact file match
if [[ "$file" == "$pattern" ]]; then
VIOLATIONS+=(" $file (blocked by: $pattern)")
fi
fi
done
done
done
if [[ ${#VIOLATIONS[@]} -gt 0 ]]; then
echo ""
echo "=========================================="
echo " PUSH BLOCKED — proprietary files detected"
echo "=========================================="
echo ""
echo "The following files must NOT be pushed to GitHub:"
echo ""
for v in "${VIOLATIONS[@]}"; do
echo "$v"
done
echo ""
echo "These paths are listed in .github-ignore."
echo ""
exit 1
fi
exit 0
+46
View File
@@ -0,0 +1,46 @@
# Paths excluded from this repository.
# Used by .githooks/pre-push to enforce boundaries (and mirrored by the
# "Proprietary Code Guard" in .github/workflows/security-check.yml).
# One path per line. Trailing slashes match directories.
# See docs/contracts/oss-plane-separation-v1.md for the open vs. private policy.
# ── Deployment / ops (private) ──
cloud/
docker-compose.yml
.gitlab-ci.yml
deploy.sh
website/
DEVELOPMENT.md
Makefile.deploy
# ── Secrets / host access (private; live credentials — gitignored, must never reach GitHub) ──
# server.md is the host-access/credentials reference (RUNBOOK §1, §10). It is
# .gitignored so it can never be staged; listed here too so the pre-push hook and
# the CI Proprietary Code Guard alarm if it is ever force-added (defense in depth).
server.md
# ── Business / monetization (private; never on the public GitHub mirror) ──
docs/business/
memory-bank/
discord-bot/
n8n-workflows/
lab/
# ── Commercial plane (lives in the private lean-ctx-cloud repo) ──
# These modules were relocated out of the open engine (oss-plane-separation-v1).
# The engine bills nothing and issues no licenses; it only emits the signed
# savings ledger. Re-adding any of these to the OSS engine is a leak —
# implement billing/licensing in lean-ctx-cloud instead.
# NB: core/billing/ stays partially OPEN (metering, plans, mod), so we list the
# specific commercial files rather than the whole directory.
# NB: core/licensing/ stays partially OPEN (mod.rs = types + Ed25519 verify,
# offline.rs = license parsing). Only keygen.rs (issuance with private key)
# is commercial and belongs in lean-ctx-cloud.
rust/src/core/license/
rust/src/core/licensing/keygen.rs
rust/src/cli/license_cmd.rs
rust/src/core/billing/success_fee.rs
rust/src/core/billing/stripe_invoice.rs
docs/contracts/license-v1.md
docs/contracts/success-fee-invoice-v1.md
+3
View File
@@ -0,0 +1,3 @@
github: [yvgude]
buy_me_a_coffee: yvgude
custom: ["https://leanctx.com/support/", "https://leanctx.com/services/"]
+28
View File
@@ -0,0 +1,28 @@
---
name: Bug Report
about: Report a bug in lean-ctx
title: 'bug: '
labels: bug
---
**lean-ctx version:** (run `lean-ctx --version`)
**OS:** (macOS / Linux / Windows)
**AI tool:** (Cursor / Claude Code / Copilot / Crush / other)
**What happened:**
**What you expected:**
**Steps to reproduce:**
1.
2.
3.
**Relevant output:**
```
(paste terminal output here)
```
@@ -0,0 +1,20 @@
---
name: New Compression Pattern
about: Request or propose a new CLI command compression pattern
title: 'pattern: '
labels: enhancement, good first issue
---
**Command:** (e.g. `mypy`, `black`, `flake8`)
**Example raw output:**
```
(paste a typical command output here)
```
**Expected compressed output:**
```
(what the compressed version should look like)
```
**Language/Ecosystem:** (Python / Rust / Go / JS / other)
+11
View File
@@ -0,0 +1,11 @@
blank_issues_enabled: false
contact_links:
- name: Discord Support
url: https://discord.gg/pTHkG9Hew9
about: Get help with setup, editors, and troubleshooting.
- name: Security Vulnerability
url: https://github.com/yvgude/lean-ctx/security/advisories/new
about: Please report security issues privately via GitHub Security Advisories.
- name: Documentation
url: https://leanctx.com/docs/getting-started
about: Start here for installation, concepts, and tool reference.
+18
View File
@@ -0,0 +1,18 @@
---
name: Feature Request
about: Suggest a new feature or improvement
title: 'feat: '
labels: enhancement
---
**Problem:**
What problem does this feature solve?
**Proposed solution:**
How should it work?
**Alternatives considered:**
What else did you consider?
**Additional context:**
Screenshots, links, related issues.
+116
View File
@@ -0,0 +1,116 @@
# ---------------------------------------------------------------------------
# windows-jemalloc — build jemalloc from source under MSYS2/MinGW
#
# WHY THIS EXISTS:
# tikv-jemalloc-sys's built-in autotools step fails on Windows CI because
# the HOST env var (x86_64-pc-windows-msvc) maps to --build=x86_64-pc-win32,
# which jemalloc 5.3.0's config.sub rejects. This action pre-builds jemalloc
# externally and uses JEMALLOC_OVERRIDE to bypass the build script entirely.
#
# PITFALLS (read before modifying):
# 1. Use the RELEASE TARBALL, not the git tag — the tag lacks configure.
# 2. Pass --with-jemalloc-prefix=_rjem_ — tikv-jemallocator expects _rjem_*.
# 3. MinGW installs jemalloc_s.lib not libjemalloc.a — rename needed.
# 4. JEMALLOC_OVERRIDE needs a Windows-native path (cygpath -w).
# 5. Provide dummy liballoc.a for -lalloc linker flag.
#
# USAGE:
# - uses: msys2/setup-msys2@v2 (first, in the caller workflow)
# - uses: dtolnay/rust-toolchain@stable (first, with targets)
# - name: Setup jemalloc for Windows
# uses: ./.github/actions/windows-jemalloc
# - name: Build
# shell: msys2 {0}
# env:
# JEMALLOC_OVERRIDE: ${{ steps.jemalloc.outputs.jemalloc-override }}
# CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS: -L ${{ steps.jemalloc.outputs.dummy-dir }}
# run: cargo build --release --target x86_64-pc-windows-gnu
# ---------------------------------------------------------------------------
name: 'Windows jemalloc'
description: Build jemalloc 5.3.0 from source under MSYS2/MinGW for Windows CI
outputs:
jemalloc-override:
description: Windows-native path to libjemalloc.a for JEMALLOC_OVERRIDE
value: ${{ steps.set-paths.outputs.jemalloc-override }}
dummy-dir:
description: Windows-native path to directory containing dummy liballoc.a
value: ${{ steps.set-paths.outputs.dummy-dir }}
runs:
using: composite
steps:
# Cache the compiled jemalloc prefix across runs — building it from
# source (configure + make) takes real minutes and never changes unless
# this action's build recipe does. Cached under the workspace (a stable,
# known Windows-native path) rather than MSYS2's /tmp, whose real
# location is an implementation detail of the msys2/setup-msys2 action.
- name: Cache jemalloc build
id: jemalloc-cache
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ github.workspace }}\.jemalloc-prefix
key: windows-jemalloc-5.3.0-${{ hashFiles('.github/actions/windows-jemalloc/action.yml') }}
# Step 1 — compile jemalloc 5.3.0 from the GitHub Release tarball
# ---------------------------------------------------------------
# Git tag 5.3.0 only has configure.ac / Makefile.in — the generated
# configure script is shipped in the release tarball exclusively.
# We must pass --with-jemalloc-prefix=_rjem_ (tikv-jemallocator expects
# _rjem_malloc symbols) and rename the MinGW .lib to the expected name.
- name: Build jemalloc 5.3.0 from release tarball
if: steps.jemalloc-cache.outputs.cache-hit != 'true'
shell: msys2 {0}
run: |
set -eux
PREFIX="$(cygpath -u "$GITHUB_WORKSPACE")/.jemalloc-prefix"
curl -fsSL https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 \
-o /tmp/jemalloc.tar.bz2
mkdir -p /tmp/jemalloc-build "$PREFIX/lib"
tar -xf /tmp/jemalloc.tar.bz2 -C /tmp/jemalloc-build --strip-components=1
cd /tmp/jemalloc-build
# config.sub from 2021 doesn't know win32* (MSYS2's build triple)
sed -i 's/^\tmingw32\* | mingw64\*/\tmingw32* | mingw64* | win32*/' build-aux/config.sub
./configure --host=x86_64-w64-mingw32 \
--disable-cxx --enable-shared=no \
--with-jemalloc-prefix=_rjem_ \
--with-private-namespace=_rjem_ \
--prefix="$PREFIX"
make -j$(nproc)
make install_lib_static install_include
# MinGW → jemalloc_s.lib; rename for JEMALLOC_OVERRIDE (strips "lib")
cp "$PREFIX/lib/jemalloc_s.lib" "$PREFIX/lib/libjemalloc.a"
# Step 2 — create a dummy liballoc.a
# -----------------------------------
# #[global_allocator] on x86_64-pc-windows-gnu causes the Rust compiler
# to emit -lalloc. The alloc crate ships as .rlib only for this target
# (no liballoc.a exists). An empty archive with a single object file
# satisfies the flag — real symbols come from the .rlib that is already
# linked statically upstream of the native-library resolution pass.
# Cheap either way, so it isn't gated on the cache hit — reruns
# harmlessly on a cache hit too.
- name: Create dummy liballoc.a for -lalloc linker flag
shell: msys2 {0}
run: |
set -eux
PREFIX="$(cygpath -u "$GITHUB_WORKSPACE")/.jemalloc-prefix"
printf 'void __dummy_alloc_ignore(void) {}\n' | \
gcc -xc -c - -o "$PREFIX/lib/dummy.o"
ar crs "$PREFIX/lib/liballoc.a" "$PREFIX/lib/dummy.o"
rm -f "$PREFIX/lib/dummy.o"
# Step 3 — export Windows-native paths for subsequent build steps
# ----------------------------------------------------------------
# The Rust build script is a native Windows process and cannot resolve
# MSYS2 virtual paths. cygpath -w produces a proper absolute Windows
# path that the native linker can open.
- name: Export Windows-native paths as outputs
id: set-paths
shell: msys2 {0}
run: |
set -eux
PREFIX="$(cygpath -u "$GITHUB_WORKSPACE")/.jemalloc-prefix"
echo "jemalloc-override=$(cygpath -w "$PREFIX/lib/libjemalloc.a")" >> $GITHUB_OUTPUT
echo "dummy-dir=$(cygpath -w "$PREFIX/lib")" >> $GITHUB_OUTPUT
+7
View File
@@ -0,0 +1,7 @@
{
"servers": {
"lean-ctx": {
"command": "lean-ctx"
}
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"hooks": {
"preToolUse": [
{
"bash": "lean-ctx hook rewrite",
"timeoutSec": 15,
"type": "command"
},
{
"bash": "lean-ctx hook redirect",
"timeoutSec": 5,
"type": "command"
}
]
},
"version": 1
}
+23
View File
@@ -0,0 +1,23 @@
## Summary
What does this PR change and why?
## Test plan
- [ ] `cd rust && cargo test`
- [ ] `cd rust && cargo clippy --all-targets --all-features -- -D warnings`
- [ ] `cd rust && cargo fmt --check`
- [ ] If cookbook/packages changed: relevant `npm test` / build steps
## Notes for reviewers
- Risk areas / edge cases:
- Backwards compatibility:
- Docs updated (links/files):
## Contributor License Agreement
First-time contributors: a bot will ask you to sign our one-time
[CLA](https://github.com/yvgude/lean-ctx/blob/main/CLA.md) (it keeps lean-ctx
Apache-2.0 and free for individual developers — see §8). You sign **once** by
replying to this PR with: `I have read the CLA Document and I hereby sign the CLA`
+211
View File
@@ -0,0 +1,211 @@
#!/usr/bin/env node
// Posts a single release-announcement tweet for @leanctx.
//
// Triggered from .github/workflows/release.yml after a stable release is
// created. Dependency-free (Node built-ins only) so the workflow needs no
// `npm install`. Auth is Twitter/X API v2 + OAuth 1.0a user context.
//
// Required env:
// TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET,
// TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_SECRET
// RELEASE_TAG e.g. "v3.6.26"
// REPO e.g. "yvgude/lean-ctx"
// Optional env:
// DRY_RUN=1 compose + print the tweet, do not post
// CHANGELOG path to changelog (default: CHANGELOG.md)
import crypto from "node:crypto";
import https from "node:https";
import { readFileSync } from "node:fs";
const MAX_TWEET = 280;
function requireEnv(name) {
const v = process.env[name];
if (!v) {
console.error(`Missing required env: ${name}`);
process.exit(1);
}
return v;
}
/**
* Pull a concise, human highlight for `version` from the changelog.
* Prefers the section's blockquote summary (the "EPIC …" one-liner); falls
* back to a factual count of Added/Fixed/Changed/Security entries.
* Returns "" when nothing meaningful is available (caller posts version+link).
*/
function extractHighlight(changelog, version) {
const lines = changelog.split(/\r?\n/);
// Match `## [<version>]` literally — `version` is data, never compiled into a
// RegExp, so there is no escaping to get wrong and no regex-injection surface.
const needle = `[${version}]`;
const start = lines.findIndex(
(l) => l.startsWith("##") && l.slice(2).trimStart().startsWith(needle),
);
if (start === -1) return "";
const section = [];
for (let i = start + 1; i < lines.length; i++) {
if (/^##\s*\[/.test(lines[i])) break;
section.push(lines[i]);
}
// Join the section's leading blockquote (the "EPIC …" summary) into one line.
const quoteLines = [];
for (const l of section) {
if (/^\s*>/.test(l)) {
quoteLines.push(l.replace(/^\s*>\s?/, ""));
} else if (quoteLines.length) {
break; // end of the contiguous blockquote block
} else if (l.trim() === "") {
continue; // skip blank lines before the quote
} else {
break; // section starts with real content -> no summary quote
}
}
if (quoteLines.length) {
const clean = quoteLines.join(" ").replace(/\*\*/g, "").replace(/\s+/g, " ").trim();
if (clean) return clean;
}
const counts = {};
let cat = null;
for (const l of section) {
const h = l.match(/^###\s+(\w+)/);
if (h) {
cat = h[1];
continue;
}
if (cat && /^\s*-\s+/.test(l)) counts[cat] = (counts[cat] || 0) + 1;
}
const order = ["Added", "Fixed", "Changed", "Security"];
const labels = {
Added: ["new feature", "new features"],
Fixed: ["fix", "fixes"],
Changed: ["change", "changes"],
Security: ["security fix", "security fixes"],
};
const parts = order
.filter((c) => counts[c])
.map((c) => `${counts[c]} ${labels[c][counts[c] === 1 ? 0 : 1]}`);
return parts.length ? parts.join(" · ") : "";
}
function composeTweet(tag, repo, highlight) {
const url = `https://github.com/${repo}/releases/tag/${tag}`;
const header = `🚀 lean-ctx ${tag} is out`;
// Twitter counts every URL as 23 chars (t.co), so reserve that, not url.length.
const urlCost = 23;
let body = header;
if (highlight) {
const budget = MAX_TWEET - header.length - urlCost - 4; // 2×"\n\n"
let h = highlight;
if (h.length > budget) h = h.slice(0, Math.max(0, budget - 1)).trimEnd() + "…";
if (h) body += `\n\n${h}`;
}
return `${body}\n\n${url}`;
}
function postTweet(text) {
const CK = requireEnv("TWITTER_CONSUMER_KEY");
const CS = requireEnv("TWITTER_CONSUMER_SECRET");
const AT = requireEnv("TWITTER_ACCESS_TOKEN");
const AS = requireEnv("TWITTER_ACCESS_SECRET");
const endpoint = "https://api.twitter.com/2/tweets";
const pct = (s) =>
encodeURIComponent(s).replace(/[!'()*]/g, (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase());
const oauth = {
oauth_consumer_key: CK,
oauth_nonce: crypto.randomBytes(16).toString("hex"),
oauth_signature_method: "HMAC-SHA1",
oauth_timestamp: Math.floor(Date.now() / 1000).toString(),
oauth_token: AT,
oauth_version: "1.0",
};
const paramStr = Object.keys(oauth)
.sort()
.map((k) => `${pct(k)}=${pct(oauth[k])}`)
.join("&");
const baseStr = `POST&${pct(endpoint)}&${pct(paramStr)}`;
const signingKey = `${pct(CS)}&${pct(AS)}`;
const signature = crypto.createHmac("sha1", signingKey).update(baseStr).digest("base64");
const authHeader =
"OAuth " +
Object.entries({ ...oauth, oauth_signature: signature })
.map(([k, v]) => `${k}="${pct(v)}"`)
.join(", ");
const payload = JSON.stringify({ text });
return new Promise((resolve, reject) => {
const req = https.request(
endpoint,
{
method: "POST",
headers: {
Authorization: authHeader,
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(payload),
},
},
(res) => {
let data = "";
res.on("data", (c) => (data += c));
res.on("end", () => resolve({ status: res.statusCode, body: data }));
},
);
req.on("error", reject);
req.write(payload);
req.end();
});
}
async function main() {
const tag = requireEnv("RELEASE_TAG");
const repo = requireEnv("REPO");
const version = tag.replace(/^v/, "");
const changelogPath = process.env.CHANGELOG || "CHANGELOG.md";
let highlight = "";
try {
highlight = extractHighlight(readFileSync(changelogPath, "utf8"), version);
} catch (e) {
console.warn(`Could not read ${changelogPath}: ${e.message} — posting version + link only`);
}
const tweet = composeTweet(tag, repo, highlight);
// Twitter weights every URL as 23 chars (t.co), regardless of real length.
const weighted = tweet.replace(/https?:\/\/\S+/g, "x".repeat(23)).length;
console.log(`Tweet (${weighted}/${MAX_TWEET} weighted chars):\n---\n${tweet}\n---`);
if (process.env.DRY_RUN === "1" || process.argv.includes("--dry-run")) {
console.log("DRY_RUN — not posting.");
return;
}
const res = await postTweet(tweet);
if (res.status === 403 && res.body.includes("duplicate content")) {
console.log("Tweet already posted (duplicate) — treating as success.");
return;
}
if (res.status !== 201) {
console.error(`Twitter API error ${res.status}: ${res.body}`);
process.exit(1);
}
let id = "";
try {
id = JSON.parse(res.body).data.id;
} catch {
/* keep id empty */
}
console.log(`Posted: https://x.com/leanctx/status/${id}`);
}
main().catch((e) => {
console.error("Fatal:", e);
process.exit(1);
});
+44
View File
@@ -0,0 +1,44 @@
name: Addon Registry Freshness
# The curated addon registry pins every upstream package to an exact version
# (supply-chain safety), but pins go stale silently. This lightweight,
# non-blocking check resolves each pin against its registry (PyPI/npm/NuGet/
# crates.io) and reports drift. It is intentionally NOT part of the `CI Green`
# gate — an upstream release must never break our own build.
on:
schedule:
- cron: "0 6 * * 1" # Mondays 06:00 UTC — weekly drift alert
workflow_dispatch:
pull_request:
branches: [main]
# Only when the registry or the checker itself changes, so unrelated PRs
# are never nagged — but a pin bump is validated the moment it is proposed.
paths:
- rust/data/addon_registry.json
- scripts/check-addon-versions.py
- .github/workflows/addon-versions.yml
permissions:
contents: read
jobs:
check:
name: Check addon pins vs upstream
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/setup-python@v5 # v5
with:
python-version: "3.11"
- name: Resolve pinned addon versions against upstream registries
shell: bash
# Warn-only on PRs / manual runs (annotations, exit 0); the weekly
# scheduled run is `--strict` so drift turns the run red as a real alert.
run: |
if [ "${{ github.event_name }}" = "schedule" ]; then
python3 scripts/check-addon-versions.py --strict
else
python3 scripts/check-addon-versions.py
fi
+680
View File
@@ -0,0 +1,680 @@
name: CI
on:
push:
branches: [main]
pull_request:
branches: [main]
workflow_dispatch:
# Cancel a still-running CI for the same ref when a new push supersedes it —
# avoids burning runner-hours on stale commits during active PR iteration.
# Never cancel on main: a run there should always finish (e.g. for the
# scorecard/coverage artifacts it produces), even if another push lands.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
RUSTFLAGS: -Dwarnings
# CI must be hermetic: never let tests download the embedding model
# (~90 MB) in the background. Without this, the #551 auto-activation can
# load the engine mid-suite and slow tarpaulin past its timeout.
LEAN_CTX_EMBEDDINGS_AUTO_DOWNLOAD: "0"
jobs:
test:
name: Test
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
# --test-threads=1 (env-race legacy, v3.3.5) serializes ~70 binaries that
# now include heavy property/fuzz/benchmark suites (#291, #493, #551) —
# well beyond 90 min on hosted runners. Parallelization is tracked
# separately; until then the gate gets the time it actually needs.
#
# Tried switching to cargo-nextest (rust/.config/nextest.toml already
# exists and documents the same env-race for 3 known binaries via a
# `serial-state` test-group). Reverted: nextest's default parallelism
# (test-threads = -4) surfaced several more tests with hidden wall-clock
# timing assumptions (e.g. core::cache::tests::hebbian_eviction_bonus_is_wired
# depends on two calls landing in the same real-time 500ms window) that a
# `max-threads=1` test-group does NOT fix — that only serializes tests
# *within* the group against each other, it doesn't protect them from
# being scheduling-delayed by the many *other* tests still running in
# parallel elsewhere on the same runner. Revisit once those tests are
# made robust to scheduling jitter (or the burst window is made
# injectable for tests) rather than papering over it with test-groups.
timeout-minutes: 180
env:
# ~60 integration-test binaries each statically link the full lib
# (tree-sitter ×18, rten, aws-lc, resvg, …). With default full debug
# info that is 25-40 GB and the ubuntu runner dies with ENOSPC / the
# linker with SIGBUS. Line tables keep file:line in panic backtraces
# at a fraction of the size.
CARGO_PROFILE_DEV_DEBUG: line-tables-only
CARGO_PROFILE_TEST_DEBUG: line-tables-only
# property_compression alone took 8012s serialized on ubuntu (87% of
# the job) with proptest's default 256 cases — and pushed Windows past
# the 180-min timeout. 64 cases keep the properties exercised on every
# push at a quarter of the cost; local runs keep the full default.
PROPTEST_CASES: 64
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- name: Ensure toolchain is active
shell: bash
run: |
CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
export PATH="$CARGO_HOME/bin:$PATH"
rustup default stable
cargo --version
echo "$CARGO_HOME/bin" >> $GITHUB_PATH
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: rust -> target
# ---- Windows: MSYS2 + GNU target + jemalloc -------------------------
# The default MSVC toolchain can't build jemalloc (autotools mismatch).
# We switch to the GNU target with MSYS2/MinGW and pre-build jemalloc.
# See .github/actions/windows-jemalloc for the gory details.
- name: Setup MSYS2 and MinGW
if: runner.os == 'Windows'
uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c # v2
with:
msystem: MINGW64
update: true
install: base-devel mingw-w64-x86_64-toolchain
path-type: inherit
- name: Install Rust GNU target
if: runner.os == 'Windows'
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
targets: x86_64-pc-windows-gnu
- name: Build jemalloc + dummy liballoc.a (Windows only)
if: runner.os == 'Windows'
id: jemalloc
uses: ./.github/actions/windows-jemalloc
- name: Free disk space
if: runner.os == 'Linux'
shell: bash
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
/opt/hostedtoolcache/CodeQL /usr/local/.ghcup /usr/share/swift \
/usr/local/share/powershell /usr/local/share/chromium \
/usr/local/share/boost /usr/lib/jvm /usr/local/julia* || true
sudo docker system prune -af --volumes >/dev/null 2>&1 || true
df -h /
- name: Run tests
working-directory: rust
shell: bash
run: |
if [[ "${{ runner.os }}" == "macOS" ]]; then
CARGO="${CARGO_HOME}/bin/cargo"
else
CARGO="cargo"
fi
if [[ "${{ runner.os }}" == "Windows" ]]; then
JEMALLOC_OVERRIDE="${{ steps.jemalloc.outputs.jemalloc-override }}" \
CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS="-L ${{ steps.jemalloc.outputs.dummy-dir }}" \
"$CARGO" test --target x86_64-pc-windows-gnu --all-features -- --test-threads=1
else
"$CARGO" test --all-features -- --test-threads=1
fi
# #581: lock in the parallel BM25 (incremental) rebuild win against silent
# regression. Timing is only meaningful on a quiet, single-threaded runner,
# so the gate is env-opt-in (no-op in the matrix above) and runs once here.
- name: BM25 build-time regression gate (#581)
if: runner.os == 'Linux'
working-directory: rust
shell: bash
env:
LEAN_CTX_PERF_GATE: "1"
run: cargo test --all-features --lib -- --test-threads=1 parallel_incremental_rebuild_perf_gate --nocapture
clippy:
name: Clippy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
components: clippy
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: rust -> target
- name: Run clippy
working-directory: rust
run: cargo clippy --all-features -- -D warnings
# Maintainability gate (#660): no Rust file grows past 1500 LOC
# (legacy files are frozen at 2000 via the script's allowlist).
- name: LOC gate (file size budget)
run: ./scripts/loc-gate.sh
# Guards the no-ORT build (#586). The `embeddings`/`neural` features pull the
# `ort` crate, whose `load-dynamic` dylib resolver has no fallback arm for
# non-tier-1 targets (FreeBSD, etc.), so community ports build with those
# features off. This job catches any code that references `ort`/embeddings
# without a feature gate. Dead code in the reduced build is expected, so
# warnings are not denied here (override the workflow-wide -Dwarnings).
build-minimal:
name: Build (no embeddings / no ORT)
runs-on: ubuntu-latest
env:
RUSTFLAGS: ""
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: rust -> target
- name: Check build without ORT-backed features
working-directory: rust
run: cargo check --no-default-features --features "tree-sitter,http-server,team-server,secure-update,jemalloc"
# The no-tree-sitter combo is the slim-binary candidate (#663) and had
# rotted silently because only the no-ORT combo was checked.
- name: Check build without tree-sitter
working-directory: rust
run: cargo check --no-default-features --features "embeddings,http-server,team-server,secure-update,jemalloc"
fmt:
name: Format
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
components: rustfmt
- name: Check formatting
working-directory: rust
run: cargo fmt --check
cookbook:
name: Cookbook (Node)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/setup-node@v4
with:
node-version: "20"
cache: "npm"
cache-dependency-path: cookbook/package-lock.json
- name: Install
working-directory: cookbook
run: npm ci
- name: Lint
working-directory: cookbook
run: npm run lint
- name: Typecheck
working-directory: cookbook
run: npm run typecheck
- name: Test
working-directory: cookbook
run: npm test
- name: Build
working-directory: cookbook
run: npm run build
pi-extension:
name: Pi Extension (Node)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
# Fail fast if pi-lean-ctx / lean-ctx-bin drifted from the engine version,
# so the Pi extension can never quietly fall behind (npm skips the publish
# of a stale version). Runs on every push/PR — not just at release.
- name: Check npm package ↔ engine version coupling
run: python3 scripts/check-package-versions.py
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: "npm"
cache-dependency-path: packages/pi-lean-ctx/package-lock.json
- name: Install
working-directory: packages/pi-lean-ctx
run: npm ci
- name: Typecheck
working-directory: packages/pi-lean-ctx
run: npm run typecheck
- name: Test
working-directory: packages/pi-lean-ctx
run: npm test
rust-sdk:
name: Rust SDK (lean-ctx-client)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
components: clippy, rustfmt
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: clients/rust/lean-ctx-client -> target
- name: Format
working-directory: clients/rust/lean-ctx-client
run: cargo fmt --check
- name: Clippy
working-directory: clients/rust/lean-ctx-client
run: cargo clippy --all-targets -- -D warnings
- name: Test
working-directory: clients/rust/lean-ctx-client
run: cargo test
- name: Docs
working-directory: clients/rust/lean-ctx-client
env:
RUSTDOCFLAGS: -D warnings
run: cargo doc --no-deps
embed-sdk:
# Track A — the in-process embedding SDK (rust/crates/lean-ctx-sdk). It is a
# workspace member but excluded from `default-members`, so the engine's own
# test/clippy/doc jobs never see it; this job is its dedicated gate
# (pedantic clippy, doctests, the engine-integration tests, example build).
name: Embed SDK (lean-ctx-sdk)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
components: clippy, rustfmt
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: rust -> target
- name: Format
working-directory: rust
run: cargo fmt -p lean-ctx-sdk --check
- name: Clippy (pedantic, 0 warnings)
working-directory: rust
run: cargo clippy -p lean-ctx-sdk --all-targets -- -D warnings
- name: Test (doctests + engine integration)
working-directory: rust
run: cargo test -p lean-ctx-sdk
- name: Example builds
working-directory: rust
run: cargo build -p lean-ctx-sdk --example embed
- name: Docs
working-directory: rust
env:
RUSTDOCFLAGS: -D warnings
run: cargo doc -p lean-ctx-sdk --no-deps
python-sdk:
name: Python SDK (leanctx)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/setup-python@v5 # v5
with:
python-version: "3.11"
- name: Install
working-directory: clients/python
run: pip install -e ".[dev]"
- name: Test
working-directory: clients/python
run: python -m pytest -q
hermes-plugin:
# The Hermes context-engine plugin (integrations/hermes-lean-ctx) replaces
# Hermes' built-in ContextCompressor with lean-ctx. Gate: pytest (ABC
# conformance + tool_call/tool_result invariant + daemon-down graceful) and
# a benchmark smoke that proves the harness runs offline (local fallback).
name: Hermes Plugin (Python)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/setup-python@v5 # v5
with:
python-version: "3.11"
- name: Install test deps
# Runtime dep (leanctx SDK) is wired onto sys.path by the plugin
# conftest from clients/python — no install needed. tiktoken makes the
# token count match the host model exactly; without it the plugin uses
# its char-based fallback (also tested).
run: pip install pytest tiktoken
- name: Run plugin tests
working-directory: integrations/hermes-lean-ctx
run: python -m pytest tests/ -q
- name: Benchmark smoke (offline, local fallback)
working-directory: integrations/hermes-lean-ctx
# Discard port → daemon unreachable → exercises deterministic local
# compaction end-to-end. Competitors are auto-skipped when absent.
run: python benchmarks/run.py --turns 40 --needles 6 --context-length 2000 --base-url http://127.0.0.1:9 --no-write
sdk-conformance:
# GL #395: every first-party SDK must prove the full /v1 contract against
# a real engine build. Red when an SDK misses a route, mistypes a shape,
# or the engine drifts from the frozen surface.
name: SDK Conformance Matrix
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: |
rust -> target
clients/rust/lean-ctx-client -> target
- uses: actions/setup-node@v4
with:
node-version: "20"
cache: "npm"
cache-dependency-path: cookbook/package-lock.json
- uses: actions/setup-python@v5 # v5
with:
python-version: "3.11"
- name: Install Python SDK
working-directory: clients/python
# langchain-core gives the adapter smoke a real framework round trip;
# LlamaIndex/CrewAI skip cleanly (heavy dep trees, covered by the
# framework-agnostic OpenAI adapter path).
run: pip install -e ".[dev]" langchain-core
- name: Run conformance kit (3 SDKs vs real server)
run: ./scripts/sdk-conformance.sh
- name: Upload conformance matrix
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: sdk-conformance-matrix
path: docs/reference/sdk-conformance-matrix.md
if-no-files-found: ignore
coverage:
name: Coverage
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
# tarpaulin's LLVM engine (below) needs llvm-profdata/llvm-cov from
# the rustup component — without it the engine silently falls back.
components: llvm-tools-preview
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: rust -> target
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL || true
df -h /
- uses: taiki-e/install-action@74e87cbfa15a59692b158178d8905a61bf6fca95 # v2
with:
tool: cargo-tarpaulin
- name: Generate coverage
working-directory: rust
env:
CARGO_BUILD_JOBS: "2"
# --engine llvm uses LLVM source-based coverage (instrument-coverage +
# llvm-profdata) instead of tarpaulin's default ptrace engine. ptrace
# is non-deterministically fragile on hosted runners: it corrupts the
# tracee's registers under load, surfacing as a bogus
# "memory allocation of 1407…bytes failed" abort followed by
# "ECHILD: No child processes" — a flake unrelated to the code (the
# Test matrix on the same SHA stays green). The LLVM engine never
# ptraces, so the whole class of flake disappears.
run: |
cargo tarpaulin --engine llvm --out xml --skip-clean --all-features --lib --timeout 180 \
--exclude-files 'src/cli/*' --exclude-files 'src/main.rs' \
-- --test-threads=1
pkill -f 'target.*lean-ctx' 2>/dev/null || true
- name: Upload to Codecov
uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
with:
fail_ci_if_error: false
deny:
name: cargo-deny
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: EmbarkStudios/cargo-deny-action@5bb39ff5d5a0e94dc9e2dc94eced0c6129743a57 # v2
with:
manifest-path: rust/Cargo.toml
adversarial:
name: Adversarial Safety
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: rust -> target
- name: Run adversarial compression tests
working-directory: rust
run: cargo test --test adversarial_compression -- --nocapture
bench:
name: Benchmarks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: rust -> target
- name: Compile benchmarks
working-directory: rust
run: cargo bench --no-run
- name: Generate reproducible scorecard
working-directory: rust
run: cargo run --quiet -- benchmark scorecard --json --output ../scorecard.json
- name: Upload scorecard artifact
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: scorecard
path: scorecard.json
quality-gate:
name: Output-Quality Gate (eval A/B)
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: rust -> target
- name: Build lean-ctx
working-directory: rust
run: cargo build --quiet
# Self-footprint gate (#578/#964): the fixed per-session cost lean-ctx
# itself injects (advertised tool schemas + MCP instructions + wakeup)
# must stay under budget. Runs in a scratch HOME + empty cwd so only OUR
# footprint is measured — no repo rules files, no developer state.
# Baseline after the diet: ~2147 tok (schemas 1685, instructions 462).
- name: Self-footprint gate (doctor overhead)
shell: bash
env:
LEAN_CTX_CONTEXT_BUDGET_TOKENS: "2600"
run: |
set -euo pipefail
BIN="$GITHUB_WORKSPACE/rust/target/debug/lean-ctx"
SCRATCH=$(mktemp -d)
mkdir -p "$SCRATCH/home" "$SCRATCH/data" "$SCRATCH/work"
cd "$SCRATCH/work"
HOME="$SCRATCH/home" LEAN_CTX_DATA_DIR="$SCRATCH/data" "$BIN" doctor overhead --gate
# Deterministic with/without output-quality proof. Three modes, in priority order:
# 1. A committed recording (rust/eval/recording.json) → byte-stable replay, no secrets,
# and the gate BLOCKS on a regression.
# 2. Otherwise, if LEAN_CTX_EVAL_* secrets are set → capture from the live model + gate.
# 3. Otherwise → skipped (the harness itself is still covered by the test job).
# Capture a recording once with: lean-ctx eval ab --suite … --record rust/eval/recording.json
- name: Run deterministic A/B output-quality gate
working-directory: rust
shell: bash
env:
LEAN_CTX_EVAL_MODEL_URL: ${{ secrets.LEAN_CTX_EVAL_MODEL_URL }}
LEAN_CTX_EVAL_MODEL: ${{ secrets.LEAN_CTX_EVAL_MODEL }}
LEAN_CTX_EVAL_MODEL_KEY: ${{ secrets.LEAN_CTX_EVAL_MODEL_KEY }}
run: |
set -euo pipefail
BIN=./target/debug/lean-ctx
REC=eval/recording.json
"$BIN" eval init eval-suite
if [ -f "$REC" ]; then
echo "Replaying committed recording (deterministic, no secrets)…"
"$BIN" eval ab --suite eval-suite/suite.ndjson --replay "$REC" --out ab-report.json --gate
elif [ -n "${LEAN_CTX_EVAL_MODEL_URL:-}" ]; then
echo "No committed recording; capturing from the configured live model…"
mkdir -p eval
"$BIN" eval ab --suite eval-suite/suite.ndjson --record "$REC" --out ab-report.json --gate
else
echo "::notice::Output-quality gate skipped — commit rust/eval/recording.json or set LEAN_CTX_EVAL_* secrets."
fi
- name: Verify artifact (signature + determinism digest)
working-directory: rust
shell: bash
run: |
if [ -f ab-report.json ]; then ./target/debug/lean-ctx eval verify ab-report.json; fi
- name: Upload signed A/B report
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: eval-ab-report
path: rust/ab-report.json
if-no-files-found: ignore
# Off-vs-on answer-quality testbench across pinned repos (#611). The committed
# local-fixture subset replays with a recorded model → byte-stable, no secrets,
# and BLOCKS on any per-repo regression. Refresh with:
# lean-ctx eval testbench --lock eval/testbench/testbench.lock.json --record eval/testbench/recording.json
- name: Run deterministic testbench gate (off vs on)
working-directory: rust
shell: bash
run: |
set -euo pipefail
BIN=./target/debug/lean-ctx
LOCK=eval/testbench/testbench.lock.json
REC=eval/testbench/recording.json
if [ -f "$REC" ]; then
echo "Replaying committed testbench subset (deterministic, no secrets)…"
"$BIN" eval testbench --lock "$LOCK" --replay "$REC" --out testbench-out --gate
else
echo "::notice::Testbench gate skipped — commit rust/eval/testbench/recording.json."
fi
- name: Upload testbench FINDINGS
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: testbench-findings
path: |
rust/testbench-out/FINDINGS.md
rust/testbench-out/regressions.json
if-no-files-found: ignore
doc:
name: Documentation
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: rust -> target
- name: Check docs compile
working-directory: rust
env:
RUSTDOCFLAGS: -D warnings
run: cargo doc --no-deps --all-features
- name: Check generated reference docs are current
working-directory: rust
run: cargo run --example gen_docs --features dev-tools -- --check
- name: Check bundled registry snapshots are canonical
working-directory: rust
run: cargo run --example gen_registry --features dev-tools -- --check
- name: Check committed testbench recording is current
working-directory: rust
run: cargo run --example gen_testbench_recording --features dev-tools -- --check
# Single aggregating gate. Point branch protection at THIS one job ("CI Green")
# instead of every matrix leg — it stays correct when jobs are added/removed and
# cannot be bypassed by a silently-missing required check. Runs always() so it can
# inspect results even when a dependency failed; fails only on failure/cancelled
# (a legitimately skipped job — e.g. the optional quality-gate — does not block).
ci-green:
name: CI Green
if: always()
needs:
- test
- clippy
- build-minimal
- fmt
- cookbook
- pi-extension
- rust-sdk
- embed-sdk
- python-sdk
- hermes-plugin
- sdk-conformance
- coverage
- deny
- adversarial
- bench
- quality-gate
- doc
runs-on: ubuntu-latest
steps:
- name: Verify no required job failed
shell: bash
env:
NEEDS_JSON: ${{ toJSON(needs) }}
run: |
echo "$NEEDS_JSON"
python3 - <<'PY'
import json, os, sys
needs = json.loads(os.environ["NEEDS_JSON"])
blocked = {k: v["result"] for k, v in needs.items()
if v["result"] in ("failure", "cancelled")}
if blocked:
print("Required CI jobs did not pass:", blocked)
sys.exit(1)
print("All required CI jobs passed (success or skipped).")
PY
# Security audit runs in security-check.yml workflow
+53
View File
@@ -0,0 +1,53 @@
name: CLA Assistant
on:
issue_comment:
types: [created]
pull_request_target:
types: [opened, closed, synchronize]
permissions:
actions: write
contents: write
pull-requests: write
statuses: write
jobs:
cla:
runs-on: ubuntu-latest
steps:
- name: CLA Assistant
# Only act on the sign/recreate comment phrases or on PR lifecycle events.
if: >-
(github.event.comment.body == 'recreate-cla' ||
github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') ||
github.event_name == 'pull_request_target'
# Pinned to the v2.6.1 commit SHA (supply-chain hardening; a moving tag
# could be repointed to malicious code). Bump deliberately when upgrading.
uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 # v2.6.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Required: a PAT (classic "repo" scope, or fine-grained with Contents +
# Pull requests write on this repo) stored as the CLA_SIGNATURES_TOKEN
# secret. It lets the action commit signatures across forked PRs.
PERSONAL_ACCESS_TOKEN: ${{ secrets.CLA_SIGNATURES_TOKEN }}
with:
path-to-document: 'https://github.com/yvgude/lean-ctx/blob/main/CLA.md'
# Signatures are stored on a dedicated branch to keep main history clean.
path-to-signatures: 'signatures/v1/cla.json'
branch: 'cla-signatures'
# Maintainer + bots never need to sign.
allowlist: 'yvgude,dependabot[bot],*[bot]'
custom-pr-sign-comment: 'I have read the CLA Document and I hereby sign the CLA'
# Friendlier, self-contained first message: explains the one-time nature,
# the §8 Local-Free guarantee, and the exact phrase — so contributors are
# not surprised by a bare red check and can sign in one reply.
custom-notsigned-prcomment: |
👋 Thanks for the contribution — we would love to merge it!
One **one-time** step remains: please sign our [CLA](https://github.com/yvgude/lean-ctx/blob/main/CLA.md). It takes ~5 seconds, you sign **once**, and every future PR is then accepted automatically. The CLA keeps lean-ctx **Apache-2.0 and free for individual developers** (guaranteed in §8) while still allowing a hosted/commercial plane.
**To sign:** reply to this PR with a comment containing exactly:
> I have read the CLA Document and I hereby sign the CLA
custom-allsigned-prcomment: 'All contributors have signed the CLA. ✅'
+65
View File
@@ -0,0 +1,65 @@
name: "CodeQL"
on:
push:
branches: [main]
pull_request:
branches: [main]
schedule:
- cron: "30 5 * * 1"
permissions:
security-events: write
contents: read
actions: read
jobs:
analyze:
name: Analyze (${{ matrix.language }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
language: [javascript-typescript, actions]
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- name: Initialize CodeQL
uses: github/codeql-action/init@ce64ddcb0d8d890d2df4a9d1c04ff297367dea2a # v3
with:
languages: ${{ matrix.language }}
queries: security-extended
- name: Autobuild
uses: github/codeql-action/autobuild@ce64ddcb0d8d890d2df4a9d1c04ff297367dea2a # v3
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@ce64ddcb0d8d890d2df4a9d1c04ff297367dea2a # v3
with:
category: "/language:${{ matrix.language }}"
analyze-rust:
name: Analyze (rust)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- name: Initialize CodeQL
uses: github/codeql-action/init@ce64ddcb0d8d890d2df4a9d1c04ff297367dea2a # v3
with:
languages: rust
- name: Build Rust
working-directory: rust
run: cargo build --release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@ce64ddcb0d8d890d2df4a9d1c04ff297367dea2a # v3
with:
category: "/language:rust"
+127
View File
@@ -0,0 +1,127 @@
# Dependency Auto-Update — ongoing patch/minor maintenance of the Cargo dependencies.
#
# Complements the one-off, manual major upgrades
# (docs/superpowers/specs/2026-06-17-dependency-upgrades-plan-a/b-design.md):
# those catch up accumulated major drift; THIS workflow then keeps
# patch + compatible minor current. It NEVER touches `--incompatible`/major —
# breaking-change bumps stay deliberately manual.
#
# Trigger: manual only (workflow_dispatch) — no cron. The maintainer
# triggers the run deliberately (e.g. after a security advisory).
#
# Token / CI gate: PR + push run via
# `${{ secrets.DEP_UPDATE_TOKEN || github.token }}`.
# IMPORTANT: a PR created with the default GITHUB_TOKEN does NOT trigger
# further workflows — the full CI suite (ci.yml: 3-OS matrix, clippy, fmt,
# deny, ...) then does NOT run automatically. Maintainer opt-in for
# automatic gating: create a fine-grained PAT as the repo secret DEP_UPDATE_TOKEN
# (contents:write + pull-requests:write), analogous to HOMEBREW_GITHUB_TOKEN.
# Without a PAT, the smoke step below guards against grossly broken PRs.
name: Dependency Auto-Update
on:
workflow_dispatch:
permissions:
contents: read
jobs:
update:
name: Compatible patch/minor update
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
components: clippy
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: rust -> target
- uses: taiki-e/install-action@74e87cbfa15a59692b158178d8905a61bf6fca95 # v2
with:
tool: cargo-edit
- name: Run compatible updates
id: update
working-directory: rust
shell: bash
run: |
set -euo pipefail
cargo upgrade --compatible
cargo update
if git diff --quiet -- Cargo.toml Cargo.lock; then
echo "changed=false" >> "$GITHUB_OUTPUT"
echo "::notice::No compatible updates available — nothing to do."
else
echo "changed=true" >> "$GITHUB_OUTPUT"
fi
- name: Smoke verify (default features)
if: steps.update.outputs.changed == 'true'
working-directory: rust
shell: bash
# relink other gh actions?
run: |
set -euo pipefail
cargo build
cargo test
cargo clippy -- -D warnings
# or set to workflow_run?
- name: Create or update pull request
if: steps.update.outputs.changed == 'true'
shell: bash
env:
GH_TOKEN: ${{ secrets.DEP_UPDATE_TOKEN || github.token }}
REPO: ${{ github.repository }}
run: |
set -euo pipefail
BRANCH="deps/auto-update"
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
BODY_FILE="$(mktemp)"
{
echo "Automated **compatible** (patch/minor) dependency update."
echo
echo "Generated by \`.github/workflows/dep-update.yml\` via"
echo "\`cargo upgrade --compatible\` + \`cargo update\`."
echo "Contains **no** incompatible/major bumps — those stay manual"
echo "(see the dependency-upgrades-plan-a/b specs)."
echo
echo '<details><summary>Cargo.lock changes</summary>'
echo
echo '```diff'
git diff -- rust/Cargo.lock | head -n 300
echo '```'
echo '</details>'
} > "$BODY_FILE"
git switch -C "$BRANCH"
git add rust/Cargo.toml rust/Cargo.lock
git commit -m "chore(deps): compatible patch/minor update"
git push --force \
"https://x-access-token:${GH_TOKEN}@github.com/${REPO}.git" \
"HEAD:${BRANCH}"
if gh pr view "$BRANCH" --json number >/dev/null 2>&1; then
gh pr edit "$BRANCH" --body-file "$BODY_FILE"
echo "::notice::Updated existing PR for $BRANCH."
else
gh pr create \
--base main \
--head "$BRANCH" \
--title "chore(deps): compatible patch/minor update" \
--body-file "$BODY_FILE"
fi
gh pr edit "$BRANCH" --add-label dependencies \
|| echo "::notice::Label 'dependencies' missing — skipped. Create it once to enable."
+256
View File
@@ -0,0 +1,256 @@
# Grammar-addon CI matrix (#690, Phase 1c).
#
# Builds each `crates/grammar-addons/<name>` cdylib for every supported
# platform, uploads the dylibs to the rolling `grammar-addons` GitHub
# Release, and regenerates `rust/data/grammar_registry.json` with the
# resulting download URLs + SHA-256 pins via a PR (same bot-commit +
# `gh pr create` pattern as dep-update.yml).
#
# The bundled registry is trusted by construction (it is compiled into the
# lean-ctx binary itself, same as data/addon_registry.json — see
# core/addons/signing.rs's doc comment) — no separate Ed25519 signing step
# is needed here; the release binary's own provenance is the trust anchor.
#
# Trigger: manual only, like dep-update.yml — publishing a new grammar
# dylib is a deliberate, reviewed action, not a per-commit side effect.
#
# Adding a new grammar: add `crates/grammar-addons/<name>` (mirror `lua/`,
# package name `grammar-<name>`) and a workspace member entry in
# rust/Cargo.toml — this workflow discovers it automatically, no edits here.
name: Grammar Addons
on:
workflow_dispatch:
inputs:
grammar:
description: 'Grammar to build (slug under crates/grammar-addons/), or "all"'
default: 'all'
required: false
permissions:
contents: read
jobs:
discover:
name: Discover grammar-addon crates
runs-on: ubuntu-latest
outputs:
grammars: ${{ steps.list.outputs.grammars }}
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- name: List crates/grammar-addons/*
id: list
shell: bash
run: |
set -euo pipefail
cd rust/crates/grammar-addons
if [[ "${{ inputs.grammar }}" == "all" || -z "${{ inputs.grammar }}" ]]; then
NAMES=$(find . -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort)
else
NAMES="${{ inputs.grammar }}"
fi
JSON=$(printf '%s\n' "$NAMES" | jq -R . | jq -sc .)
echo "grammars=${JSON}" >> "$GITHUB_OUTPUT"
build:
name: Build ${{ matrix.grammar }} (${{ matrix.target }})
needs: discover
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
grammar: ${{ fromJson(needs.discover.outputs.grammars) }}
include:
- target: x86_64-unknown-linux-gnu
os: ubuntu-22.04
ext: so
- target: aarch64-unknown-linux-gnu
os: ubuntu-22.04
ext: so
cross: true
- target: x86_64-apple-darwin
os: macos-latest
ext: dylib
- target: aarch64-apple-darwin
os: macos-latest
ext: dylib
- target: x86_64-pc-windows-msvc
os: windows-latest
ext: dll
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
targets: ${{ matrix.target }}
- name: Install cross-compilation tools
if: matrix.cross == true
run: |
sudo apt-get update
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> $GITHUB_ENV
- name: Build
working-directory: rust
shell: bash
run: cargo build --release --locked -p "grammar-${{ matrix.grammar }}" --target ${{ matrix.target }}
- name: Rename + hash
id: asset
working-directory: rust
shell: bash
run: |
set -euo pipefail
crate_underscored="grammar_$(echo '${{ matrix.grammar }}' | tr '-' '_')"
src="target/${{ matrix.target }}/release/${crate_underscored}.${{ matrix.ext }}"
name="${{ matrix.grammar }}-${{ matrix.target }}.${{ matrix.ext }}"
cp "$src" "$name"
sha256sum "$name" > "${name}.sha256"
echo "name=${name}" >> "$GITHUB_OUTPUT"
# ABI version is a property of the grammar dylib itself (which
# tree-sitter core/tree-sitter-language version it was built against),
# not of the target platform — so it only needs deriving once per
# grammar. Native (non-cross) x86_64-linux is the one leg guaranteed
# to be able to run the dylib it just built, generically for any
# grammar via grammar-dlopen-host's --abi-only mode.
- name: Derive abi_version (native leg only)
if: matrix.target == 'x86_64-unknown-linux-gnu'
id: abi
working-directory: rust
shell: bash
run: |
set -euo pipefail
cargo build --release --locked -p grammar-dlopen-host --target ${{ matrix.target }}
out="$(./target/${{ matrix.target }}/release/grammar-dlopen-host \
"${{ steps.asset.outputs.name }}" --abi-only)"
echo "$out" > "${{ matrix.grammar }}.abi"
echo "${out#abi_version=}"
- name: Upload artifact
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: ${{ steps.asset.outputs.name }}
path: |
rust/${{ steps.asset.outputs.name }}
rust/${{ steps.asset.outputs.name }}.sha256
rust/${{ matrix.grammar }}.abi
publish:
name: Publish release assets + registry PR
needs: build
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
path: artifacts
merge-multiple: true
- name: Upload dylibs to the rolling grammar-addons release
env:
GH_TOKEN: ${{ secrets.DEP_UPDATE_TOKEN || github.token }}
run: |
set -euo pipefail
if ! gh release view grammar-addons --repo "${{ github.repository }}" >/dev/null 2>&1; then
gh release create grammar-addons --repo "${{ github.repository }}" \
--title "Grammar addons" \
--notes "Rolling release: per-platform grammar-addon dylibs referenced by rust/data/grammar_registry.json. Not a lean-ctx version release." \
--prerelease
fi
shopt -s nullglob
for f in artifacts/*.dll artifacts/*.dylib artifacts/*.so; do
gh release upload grammar-addons "$f" --repo "${{ github.repository }}" --clobber
done
- name: Regenerate rust/data/grammar_registry.json
id: regen
shell: bash
run: |
set -euo pipefail
RELEASE_URL="https://github.com/${{ github.repository }}/releases/download/grammar-addons"
for dir in rust/crates/grammar-addons/*/; do
grammar="$(basename "$dir")"
crate="grammar-${grammar}"
version="$(grep -m1 '^version' "$dir/Cargo.toml" | sed -E 's/.*"([^"]+)".*/\1/')"
license="$(grep -m1 '^license' "$dir/Cargo.toml" | sed -E 's/.*"([^"]+)".*/\1/')"
# Derived on the native x86_64-linux build leg (see "Derive
# abi_version" step) — not a platform-specific value, so any one
# leg's reading is authoritative for every asset below.
abi_file="artifacts/${grammar}.abi"
if [ ! -f "$abi_file" ]; then
echo "::error::no ${abi_file} — did the x86_64-unknown-linux-gnu leg run for '${grammar}'?" >&2
exit 1
fi
abi_version="$(sed -E 's/^abi_version=//' "$abi_file")"
assets="{}"
for shafile in artifacts/"${grammar}"-*.sha256; do
[ -e "$shafile" ] || continue
filename="$(basename "${shafile%.sha256}")"
sha="$(awk '{print $1}' "$shafile")"
target="${filename#"${grammar}"-}"
target="${target%.*}"
assets=$(jq --arg t "$target" --arg f "$filename" --arg u "${RELEASE_URL}/${filename}" --arg s "$sha" \
'.[$t] = {filename: $f, url: $u, sha256: $s}' <<<"$assets")
done
jq -n --arg name "$grammar" --arg version "$version" --arg license "$license" \
--argjson abi_version "$abi_version" --argjson assets "$assets" \
'{name: $name, display_name: ($name | ascii_upcase[0:1] + $name[1:]), version: $version, license: $license, extensions: [$name], abi_version: $abi_version, assets: $assets}' \
> "/tmp/${grammar}.json"
done
jq -s '{"$schema": "https://leanctx.dev/schema/grammar-registry-v1.json", registry_version: 1, grammars: .}' \
/tmp/*.json > rust/data/grammar_registry.json.new
mv rust/data/grammar_registry.json.new rust/data/grammar_registry.json
if git diff --quiet -- rust/data/grammar_registry.json; then
echo "changed=false" >> "$GITHUB_OUTPUT"
else
echo "changed=true" >> "$GITHUB_OUTPUT"
fi
- name: Create or update pull request
if: steps.regen.outputs.changed == 'true'
shell: bash
env:
GH_TOKEN: ${{ secrets.DEP_UPDATE_TOKEN || github.token }}
REPO: ${{ github.repository }}
run: |
set -euo pipefail
BRANCH="grammar-addons/registry-update"
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git switch -C "$BRANCH"
git add rust/data/grammar_registry.json
git commit -m "chore(grammar-addons): regenerate grammar_registry.json"
git push --force \
"https://x-access-token:${GH_TOKEN}@github.com/${REPO}.git" \
"HEAD:${BRANCH}"
BODY="Regenerated by .github/workflows/grammar-addons.yml after building/publishing the grammar-addon dylibs. Review the diff before merging — this changes what lean-ctx's next release will fetch and dlopen."
if gh pr view "$BRANCH" --json number >/dev/null 2>&1; then
gh pr edit "$BRANCH" --body "$BODY"
else
gh pr create --base main --head "$BRANCH" \
--title "chore(grammar-addons): regenerate grammar_registry.json" \
--body "$BODY"
fi
+47
View File
@@ -0,0 +1,47 @@
# `/reopen` command (GH #388): GitHub only lets issue authors reopen issues
# they closed themselves — once a maintainer closes, the author is locked out
# and ends up filing duplicates. This workflow restores that ability: the
# original author comments `/reopen` and the issue reopens automatically.
#
# Matching is deliberately relaxed (`contains`, not `startsWith`): people
# write "Please /reopen" or bury the command mid-comment (GH #388 feedback).
# False positives are bounded — only the original author on their own closed
# issue can trigger it, and the worst case is an issue reopening.
#
# Issues closed as "not planned" are excluded on purpose: that close state is
# a maintainer decision, so the command answers with a hint instead.
#
# Security: no checkout, comment body is only used in the `if` expression
# (never interpolated into shell), token scope is issues:write only.
name: Issue reopen command
on:
issue_comment:
types: [created]
permissions:
issues: write
jobs:
reopen:
if: >-
!github.event.issue.pull_request &&
github.event.issue.state == 'closed' &&
github.event.comment.user.login == github.event.issue.user.login &&
contains(github.event.comment.body, '/reopen')
runs-on: ubuntu-latest
steps:
- name: Reopen issue (or explain why not)
env:
GH_TOKEN: ${{ github.token }}
ISSUE: ${{ github.event.issue.number }}
run: |
set -euo pipefail
reason=$(gh api "repos/$GITHUB_REPOSITORY/issues/$ISSUE" --jq '.state_reason // "completed"')
if [ "$reason" = "not_planned" ]; then
gh issue comment "$ISSUE" --repo "$GITHUB_REPOSITORY" --body \
"This issue was closed as **not planned**, which is a maintainer decision, so \`/reopen\` does not apply here. A maintainer will review this request."
else
gh issue reopen "$ISSUE" --repo "$GITHUB_REPOSITORY" \
--comment "Reopened at the author's request via \`/reopen\`."
fi
+107
View File
@@ -0,0 +1,107 @@
name: JetBrains Plugin
on:
push:
branches: [ main ]
paths:
- 'packages/jetbrains-lean-ctx/**'
- '.github/workflows/jetbrains-plugin.yml'
pull_request:
branches: [ main ]
paths:
- 'packages/jetbrains-lean-ctx/**'
- '.github/workflows/jetbrains-plugin.yml'
# Release packaging lives in release.yml (the `jetbrains-plugin` job attaches the
# plugin .zip to each GitHub Release). This workflow is CI-only: validate + build +
# test on plugin source changes.
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
defaults:
run:
working-directory: packages/jetbrains-lean-ctx
jobs:
actionlint:
name: Actionlint
runs-on: ubuntu-latest
timeout-minutes: 5
defaults:
run:
working-directory: .
steps:
- name: Fetch Sources
uses: actions/checkout@v6
with:
persist-credentials: false
- name: Run actionlint
run: |
bash <(curl -sSf https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
./actionlint -color .github/workflows/jetbrains-plugin.yml
validation:
name: Validation
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v6
- uses: gradle/actions/wrapper-validation@3f131e8634966bd73d06cc69884922b02e6faf92 # v6
build:
name: Build
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- name: Fetch Sources
uses: actions/checkout@v6
with:
persist-credentials: false
- name: Setup Java
uses: actions/setup-java@v5
with:
distribution: zulu
java-version: 21
- name: Setup Gradle
uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6
- name: Build plugin
run: ./gradlew buildPlugin --console=plain
test:
name: Test
needs: [ build ]
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Fetch Sources
uses: actions/checkout@v6
with:
persist-credentials: false
- name: Setup Java
uses: actions/setup-java@v5
with:
distribution: zulu
java-version: 21
- name: Setup Gradle
uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6
with:
gradle-home-cache-cleanup: true
- name: Run Tests
run: ./gradlew check --console=plain
- name: Collect Tests Result
if: ${{ failure() }}
uses: actions/upload-artifact@v7
with:
name: jetbrains-tests-result
path: packages/jetbrains-lean-ctx/build/reports/tests
+101
View File
@@ -0,0 +1,101 @@
name: Publish /v1 clients
# Publishes the thin /v1 contract clients under one consistent name `lean-ctx-client`:
# - npm -> cookbook/sdk (TypeScript)
# - PyPI -> clients/python (Python; import module stays `leanctx`)
# - crates -> clients/rust/... (Rust)
# Versioned independently of the engine, so NOT part of release.yml. Trigger manually
# or push a `client-v*` tag. Every step is idempotent: an already-published version is a no-op.
on:
workflow_dispatch:
inputs:
targets:
description: "Which registries to publish to"
type: choice
default: npm
options: [all, npm, pypi, crates]
push:
tags:
- 'client-v[0-9]*'
permissions:
contents: read
jobs:
publish-npm:
name: Publish lean-ctx-client to npm
if: ${{ github.event_name == 'push' || inputs.targets == 'all' || inputs.targets == 'npm' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: 22
registry-url: https://registry.npmjs.org
- name: Build, test, publish
working-directory: cookbook
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
set -euo pipefail
npm ci
npm --workspace lean-ctx-client run build
npm --workspace lean-ctx-client test
PKG_VERSION="$(node -p "require('./sdk/package.json').version")"
if npm view "lean-ctx-client@${PKG_VERSION}" version >/dev/null 2>&1; then
echo "npm already has lean-ctx-client@${PKG_VERSION} — skipping publish."
exit 0
fi
npm publish --workspace lean-ctx-client --access public
publish-pypi:
name: Publish lean-ctx-client to PyPI
if: ${{ github.event_name == 'push' || inputs.targets == 'all' || inputs.targets == 'pypi' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Build and upload (skip if no token / already present)
working-directory: clients/python
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
run: |
set -euo pipefail
if [ -z "${TWINE_PASSWORD}" ]; then
echo "PYPI_TOKEN secret not set — skipping PyPI publish."
exit 0
fi
python -m pip install --quiet build twine
python -m build
python -m twine upload --skip-existing dist/*
publish-crates:
name: Publish lean-ctx-client to crates.io
if: ${{ github.event_name == 'push' || inputs.targets == 'all' || inputs.targets == 'crates' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- name: Publish (idempotent)
working-directory: clients/rust/lean-ctx-client
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
run: |
set -euo pipefail
CRATE_VERSION="$(grep -m1 '^version' Cargo.toml | sed -E 's/.*"([^"]+)".*/\1/')"
if curl -fsSL -H "User-Agent: lean-ctx-release" \
"https://crates.io/api/v1/crates/lean-ctx-client/${CRATE_VERSION}" >/dev/null 2>&1; then
echo "crates.io already has lean-ctx-client ${CRATE_VERSION} — skipping publish."
exit 0
fi
cargo publish --token "${CARGO_REGISTRY_TOKEN}"
+76
View File
@@ -0,0 +1,76 @@
name: Publish SDK
# Publishes the drop-in compress() SDKs (packages/node-lean-ctx -> npm `lean-ctx-sdk`,
# packages/python-lean-ctx -> PyPI `lean-ctx-sdk`). These versions independently of the
# engine, so they are NOT part of release.yml (engine `v*` tags). Trigger manually, or
# push an `sdk-v*` tag. Every step is idempotent: an already-published version is a no-op.
on:
workflow_dispatch:
inputs:
targets:
description: "Which registries to publish to"
type: choice
default: both
options: [both, npm, pypi]
push:
tags:
- 'sdk-v[0-9]*'
permissions:
contents: read
jobs:
publish-npm:
name: Publish lean-ctx-sdk to npm
if: ${{ github.event_name == 'push' || inputs.targets == 'both' || inputs.targets == 'npm' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: 22
registry-url: https://registry.npmjs.org
- name: Build, test, publish
working-directory: packages/node-lean-ctx
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
set -euo pipefail
npm ci
npm run build
npm test
PKG_VERSION="$(node -p "require('./package.json').version")"
if npm view "lean-ctx-sdk@${PKG_VERSION}" version >/dev/null 2>&1; then
echo "npm already has lean-ctx-sdk@${PKG_VERSION} — skipping publish."
exit 0
fi
npm publish --access public
publish-pypi:
name: Publish lean-ctx-sdk to PyPI
if: ${{ github.event_name == 'push' || inputs.targets == 'both' || inputs.targets == 'pypi' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Build and upload (skip if no token / already present)
working-directory: packages/python-lean-ctx
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
run: |
set -euo pipefail
if [ -z "${TWINE_PASSWORD}" ]; then
echo "PYPI_TOKEN secret not set — skipping PyPI publish."
exit 0
fi
python -m pip install --quiet build twine
python -m build
python -m twine upload --skip-existing dist/*
+93
View File
@@ -0,0 +1,93 @@
name: Publish VS Code Extension
# Decoupled from the binary `Release` workflow so the editor extension ships on
# its own cadence. Trigger either by pushing a `vscode-v*` tag (e.g. vscode-v0.1.0)
# or manually via the Actions tab with an explicit version.
on:
push:
tags:
- 'vscode-v*'
workflow_dispatch:
inputs:
version:
description: 'Extension version to publish (e.g. 0.1.0). Defaults to package.json.'
required: false
type: string
permissions:
contents: write
jobs:
publish-vscode:
name: Package & publish (Marketplace + Open VSX)
runs-on: ubuntu-latest
defaults:
run:
working-directory: vscode-extension
env:
# Step-level `if` guards read these so the job still packages + uploads the
# .vsix even before the publishing tokens are configured as repo secrets.
VSCE_PAT: ${{ secrets.VSCE_PAT }}
OVSX_PAT: ${{ secrets.OVSX_PAT }}
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: 22
- name: Install dependencies
run: npm install
- name: Resolve version
id: ver
run: |
if [ -n "${{ inputs.version }}" ]; then
VERSION="${{ inputs.version }}"
elif [[ "${GITHUB_REF_NAME}" == vscode-v* ]]; then
VERSION="${GITHUB_REF_NAME#vscode-v}"
else
VERSION="$(node -p "require('./package.json').version")"
fi
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
npm version "${VERSION}" --no-git-tag-version --allow-same-version
- name: Compile
run: npm run compile
- name: Package .vsix
run: npx @vscode/vsce package --no-dependencies -o "${GITHUB_WORKSPACE}/lean-ctx-${{ steps.ver.outputs.version }}.vsix"
- name: Publish to VS Code Marketplace
if: ${{ env.VSCE_PAT != '' }}
run: npx @vscode/vsce publish --packagePath "${GITHUB_WORKSPACE}/lean-ctx-${{ steps.ver.outputs.version }}.vsix" -p "${VSCE_PAT}"
- name: Publish to Open VSX (Cursor / VSCodium / Windsurf)
if: ${{ env.OVSX_PAT != '' }}
run: npx ovsx publish "${GITHUB_WORKSPACE}/lean-ctx-${{ steps.ver.outputs.version }}.vsix" -p "${OVSX_PAT}"
- name: Upload .vsix artifact
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: lean-ctx-vsix
path: lean-ctx-*.vsix
- name: Attach .vsix to GitHub release
if: ${{ startsWith(github.ref, 'refs/tags/vscode-v') }}
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
with:
tag_name: ${{ github.ref_name }}
# Never let the extension release become the repo's "latest" — install.sh
# and the npm postinstall resolve binaries via /releases/latest, so a
# vscode-v* release hijacking "latest" breaks CLI installs (see #345).
make_latest: false
name: lean-ctx VS Code extension ${{ steps.ver.outputs.version }}
files: lean-ctx-*.vsix
body: |
lean-ctx VS Code extension **${{ steps.ver.outputs.version }}**.
Install from the [VS Code Marketplace](https://marketplace.visualstudio.com/items?itemName=LeanCTX.lean-ctx)
or [Open VSX](https://open-vsx.org/extension/LeanCTX/lean-ctx) (Cursor, VSCodium, Windsurf),
or download the `.vsix` below and run `code --install-extension lean-ctx-${{ steps.ver.outputs.version }}.vsix`.
+484
View File
@@ -0,0 +1,484 @@
name: Release
on:
push:
tags:
# Only true semver release tags (v1.2.3). The digit class deliberately
# excludes the editor-extension tags (`vscode-v*`), which have their own
# workflow (publish-vscode.yml) and must NOT trigger a binary release.
- 'v[0-9]*'
permissions:
contents: write
jobs:
sdk-gate:
# GL #395: an engine release must never ship while a first-party SDK
# cannot speak its http_mcp contract version. Minor-version drift across
# the SDK family surfaces as a warning annotation.
#
# Also gates the engine-coupled npm wrappers (pi-lean-ctx, lean-ctx-bin):
# a stale package.json would let the publish step silently skip, shipping a
# release whose Pi extension / npx wrapper lags the engine.
name: Release gates (SDK + package versions)
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- name: Check SDK ↔ engine coupling
run: python3 scripts/check-sdk-versions.py
- name: Check npm package ↔ engine version coupling
run: python3 scripts/check-package-versions.py
build:
name: Build ${{ matrix.artifact || matrix.target }}
needs: sdk-gate
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- target: x86_64-unknown-linux-gnu
os: ubuntu-22.04
- target: x86_64-unknown-linux-gnu
artifact: x86_64-unknown-linux-gnu-cuda
os: ubuntu-22.04
cargo_features: ort-cuda
- target: aarch64-unknown-linux-gnu
os: ubuntu-22.04
cross: true
- target: x86_64-unknown-linux-musl
os: ubuntu-22.04
musl: true
- target: aarch64-unknown-linux-musl
os: ubuntu-22.04
musl: true
cross: true
zig: true
- target: x86_64-apple-darwin
os: macos-latest
- target: aarch64-apple-darwin
os: macos-latest
- target: x86_64-pc-windows-msvc
os: windows-latest
- target: x86_64-pc-windows-gnu
os: windows-latest
jemalloc: true
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
targets: ${{ matrix.target }}
- name: Install musl tools
if: matrix.musl == true && matrix.zig != true
run: |
sudo apt-get update
sudo apt-get install -y musl-tools
if [[ "${{ matrix.target }}" == aarch64-* ]]; then
TOOLCHAIN_ROOT="${RUNNER_TEMP}/aarch64-linux-musl-cross"
curl -sL "https://musl.cc/aarch64-linux-musl-cross.tgz" | tar xz -C "${RUNNER_TEMP}"
echo "${TOOLCHAIN_ROOT}/bin" >> $GITHUB_PATH
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-musl-gcc" >> $GITHUB_ENV
echo "CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc" >> $GITHUB_ENV
echo "AR_aarch64_unknown_linux_musl=aarch64-linux-musl-ar" >> $GITHUB_ENV
echo "RANLIB_aarch64_unknown_linux_musl=aarch64-linux-musl-ranlib" >> $GITHUB_ENV
fi
- name: Install cross-compilation tools
if: matrix.cross == true && matrix.musl != true
run: |
sudo apt-get update
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> $GITHUB_ENV
# ---- Windows jemalloc target: MSYS2 + pre-built jemalloc ------------
# See .github/actions/windows-jemalloc for the pitfall details.
- name: Setup MSYS2 and MinGW
if: matrix.jemalloc == true
uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c # v2
with:
msystem: MINGW64
update: true
install: base-devel mingw-w64-x86_64-toolchain
path-type: inherit
- name: Build jemalloc (Windows GNU target)
if: matrix.jemalloc == true
id: jemalloc
uses: ./.github/actions/windows-jemalloc
- name: Run tests
if: matrix.cross != true
working-directory: rust
shell: bash
run: |
if [[ "${{ matrix.jemalloc }}" == "true" ]]; then
JEMALLOC_OVERRIDE="${{ steps.jemalloc.outputs.jemalloc-override }}" \
CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS="-L ${{ steps.jemalloc.outputs.dummy-dir }}" \
cargo test --lib --target x86_64-pc-windows-gnu --all-features --locked -- --test-threads=1
else
cargo test --lib --all-features --locked -- --test-threads=1
fi
- name: Setup Zig
if: matrix.zig == true
run: |
pip3 install -q ziglang==0.13.0
ZIG_BIN="$(python3 -c 'import ziglang, os; print(os.path.dirname(ziglang.__file__))')"
echo "${ZIG_BIN}" >> $GITHUB_PATH
"${ZIG_BIN}/zig" version
- name: Install cargo-zigbuild
if: matrix.zig == true
uses: taiki-e/install-action@74e87cbfa15a59692b158178d8905a61bf6fca95 # v2
with:
tool: cargo-zigbuild
# Ship the production feature set (Cargo default features incl. secure-update),
# NOT --all-features — the latter would pull dev-only tooling (dev-tools:
# gen_docs/gen_mcp_manifest) into the released binary. Default features are
# already exercised by the CI test job (which runs the --all-features superset).
# --locked pins Cargo.lock so releases are reproducible (no silent dep drift).
- name: Build
working-directory: rust
shell: bash
run: |
features="${{ matrix.cargo_features }}"
feature_args=()
if [[ -n "$features" ]]; then
feature_args+=(--features "$features")
fi
if [[ "${{ matrix.zig }}" == "true" ]]; then
cargo zigbuild --release --locked --target ${{ matrix.target }} "${feature_args[@]}"
elif [[ "${{ matrix.jemalloc }}" == "true" ]]; then
JEMALLOC_OVERRIDE="${{ steps.jemalloc.outputs.jemalloc-override }}" \
CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS="-L ${{ steps.jemalloc.outputs.dummy-dir }}" \
cargo build --release --locked --target ${{ matrix.target }} "${feature_args[@]}"
else
cargo build --release --locked --target ${{ matrix.target }} "${feature_args[@]}"
fi
- name: Package (Unix)
if: runner.os != 'Windows'
shell: bash
run: |
artifact="${{ matrix.artifact || matrix.target }}"
cd rust/target/${{ matrix.target }}/release
tar czf ../../../../lean-ctx-${artifact}.tar.gz lean-ctx
cd ../../../..
- name: Package (Windows)
if: runner.os == 'Windows'
shell: pwsh
run: |
Compress-Archive -Path "rust/target/${{ matrix.target }}/release/lean-ctx.exe" -DestinationPath "lean-ctx-${{ matrix.artifact || matrix.target }}.zip"
- name: Upload artifact
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: lean-ctx-${{ matrix.artifact || matrix.target }}
path: |
lean-ctx-${{ matrix.artifact || matrix.target }}.tar.gz
lean-ctx-${{ matrix.artifact || matrix.target }}.zip
release:
name: Create Release
needs: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
merge-multiple: true
- name: Create stable source tarball
run: |
VERSION="${GITHUB_REF_NAME#v}"
git archive --format=tar.gz --prefix="lean-ctx-${VERSION}/" "${GITHUB_REF_NAME}" -o "lean-ctx-${VERSION}-source.tar.gz"
- name: Generate checksums
run: sha256sum lean-ctx-*.tar.gz lean-ctx-*.zip > SHA256SUMS
- name: Extract release notes from CHANGELOG
id: notes
run: |
VERSION="${GITHUB_REF_NAME#v}"
PREV_TAG=$(git tag --sort=-v:refname | grep -A1 "^v${VERSION}$" | tail -1)
NOTES=$(awk "/^## \\[${VERSION}\\]/{found=1; next} /^## \\[/{if(found) exit} found{print}" CHANGELOG.md)
{
echo "body<<BODY_EOF"
echo "$NOTES"
echo ""
echo "#### Upgrade"
echo '```bash'
echo "lean-ctx update # recommended (auto-downloads + refreshes shell hooks)"
echo "cargo install lean-ctx # or"
echo "npm update -g lean-ctx-bin # or"
echo "brew upgrade lean-ctx"
echo '```'
echo ""
echo "> **Note:** After upgrading via cargo/npm/brew, run \`lean-ctx setup\` to refresh shell aliases. \`lean-ctx update\` does this automatically."
echo ""
echo "**Full Changelog**: https://github.com/yvgude/lean-ctx/compare/${PREV_TAG}...v${VERSION}"
echo "BODY_EOF"
} >> "$GITHUB_OUTPUT"
- name: Create GitHub Release
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
with:
body: ${{ steps.notes.outputs.body }}
files: |
lean-ctx-*.tar.gz
lean-ctx-*.zip
SHA256SUMS
jetbrains-plugin:
# Build the JetBrains/IntelliJ plugin and attach it to the GitHub Release as a
# downloadable .zip (#418). The plugin's own workflow only runs on plugin source
# changes; the GITHUB_TOKEN-created release here does not trigger other workflows,
# so the asset must be produced from inside this release pipeline. `-Pversion`
# mirrors the tag so the plugin version never drifts from the engine release.
name: Attach JetBrains Plugin
needs: release
runs-on: ubuntu-latest
if: ${{ !contains(github.ref_name, '-rc') }}
permissions:
contents: write
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- name: Setup Java
uses: actions/setup-java@v5
with:
distribution: zulu
java-version: 21
- name: Setup Gradle
uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6
- name: Build plugin distribution
working-directory: packages/jetbrains-lean-ctx
run: ./gradlew buildPlugin -Pversion="${GITHUB_REF_NAME#v}" --console=plain
- name: Attach plugin ZIP to release
env:
GH_TOKEN: ${{ github.token }}
# Gradle's rootProject.name is "lean-ctx", so buildPlugin emits
# build/distributions/lean-ctx-<version>.zip — indistinguishable from a
# source archive in the release asset list, which is why the plugin
# looked "missing" (#418). Re-name it to a discoverable, unambiguous
# asset before upload so users can actually find the JetBrains plugin.
run: |
VERSION="${GITHUB_REF_NAME#v}"
SRC="$(ls packages/jetbrains-lean-ctx/build/distributions/*.zip | head -1)"
if [ -z "$SRC" ]; then
echo "::error::No plugin distribution zip produced by buildPlugin" >&2
exit 1
fi
DEST="lean-ctx-jetbrains-plugin-${VERSION}.zip"
cp "$SRC" "$DEST"
gh release upload "${GITHUB_REF_NAME}" "$DEST" --clobber
publish-crates:
name: Publish to crates.io
needs: release
runs-on: ubuntu-latest
if: ${{ !contains(github.ref_name, '-rc') }}
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
# Fail the release if publishing fails — a silent skip used to ship a GitHub
# release whose crates.io artifact was missing. Re-runs are idempotent: an
# already-published version is treated as a no-op, not a failure.
- name: Publish
working-directory: rust
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
run: |
set -euo pipefail
CRATE_VERSION="$(grep -m1 '^version' Cargo.toml | sed -E 's/.*"([^"]+)".*/\1/')"
if curl -fsSL -H "User-Agent: lean-ctx-release" \
"https://crates.io/api/v1/crates/lean-ctx/${CRATE_VERSION}" >/dev/null 2>&1; then
echo "crates.io already has lean-ctx ${CRATE_VERSION} — skipping publish."
exit 0
fi
# --no-verify: rmcp 2.2.0 on crates.io has a compile bug (from_bytes_stream
# vs from_byte_stream); our [patch.crates-io] fixes it but is stripped by
# cargo package. Binary builds use --locked and are unaffected.
cargo publish --allow-dirty --no-verify --token "${CARGO_REGISTRY_TOKEN}"
publish-npm:
name: Publish to npm
needs: release
runs-on: ubuntu-latest
if: ${{ !contains(github.ref_name, '-rc') }}
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: 22
registry-url: https://registry.npmjs.org
# Fail-hard on publish errors; idempotent on an already-published version
# (npm view exits non-zero when the exact version is absent). The version is
# read from package.json so the guard always matches what npm will publish.
- name: Publish lean-ctx-bin
working-directory: packages/lean-ctx-bin
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
set -euo pipefail
PKG_VERSION="$(node -p "require('./package.json').version")"
if npm view "lean-ctx-bin@${PKG_VERSION}" version >/dev/null 2>&1; then
echo "npm already has lean-ctx-bin@${PKG_VERSION} — skipping publish."
exit 0
fi
npm publish --access public
- name: Publish pi-lean-ctx
working-directory: packages/pi-lean-ctx
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
set -euo pipefail
PKG_VERSION="$(node -p "require('./package.json').version")"
if npm view "pi-lean-ctx@${PKG_VERSION}" version >/dev/null 2>&1; then
echo "npm already has pi-lean-ctx@${PKG_VERSION} — skipping publish."
exit 0
fi
# prepack builds the self-contained vendor bundle (esbuild devDep),
# so the published package ships with zero runtime dependencies (#670).
npm ci
npm publish --access public
update-homebrew:
name: Update Homebrew
needs: release
runs-on: ubuntu-latest
if: ${{ !contains(github.ref_name, '-rc') }}
steps:
- name: Download checksums and extract platform hashes
id: checksums
run: |
VERSION="${GITHUB_REF_NAME#v}"
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
gh release download "${GITHUB_REF_NAME}" \
--repo yvgude/lean-ctx --pattern "SHA256SUMS" --dir .
AARCH64_DARWIN=$(grep "aarch64-apple-darwin\.tar\.gz" SHA256SUMS | awk '{print $1}')
X86_64_DARWIN=$(grep "x86_64-apple-darwin\.tar\.gz" SHA256SUMS | awk '{print $1}')
AARCH64_LINUX_GNU=$(grep "aarch64-unknown-linux-gnu\.tar\.gz" SHA256SUMS | awk '{print $1}')
X86_64_LINUX_GNU=$(grep "x86_64-unknown-linux-gnu\.tar\.gz" SHA256SUMS | awk '{print $1}')
echo "aarch64_darwin_sha=${AARCH64_DARWIN}" >> "$GITHUB_OUTPUT"
echo "x86_64_darwin_sha=${X86_64_DARWIN}" >> "$GITHUB_OUTPUT"
echo "aarch64_linux_sha=${AARCH64_LINUX_GNU}" >> "$GITHUB_OUTPUT"
echo "x86_64_linux_sha=${X86_64_LINUX_GNU}" >> "$GITHUB_OUTPUT"
env:
GH_TOKEN: ${{ github.token }}
- name: Clone and generate binary formula
run: |
VERSION="${{ steps.checksums.outputs.version }}"
AARCH64_DARWIN_SHA="${{ steps.checksums.outputs.aarch64_darwin_sha }}"
X86_64_DARWIN_SHA="${{ steps.checksums.outputs.x86_64_darwin_sha }}"
AARCH64_LINUX_SHA="${{ steps.checksums.outputs.aarch64_linux_sha }}"
X86_64_LINUX_SHA="${{ steps.checksums.outputs.x86_64_linux_sha }}"
git clone "https://x-access-token:${HOMEBREW_TOKEN}@github.com/yvgude/homebrew-lean-ctx.git"
cd homebrew-lean-ctx
cat > Formula/lean-ctx.rb <<EOF
class LeanCtx < Formula
desc "The Context Engineering Layer for AI Coding — 71 MCP tools, 10 read modes, 95+ shell patterns"
homepage "https://leanctx.com"
version "${VERSION}"
license "Apache-2.0"
# Semantic search (ctx_semantic_search / embeddings) loads
# libonnxruntime at runtime; the engine resolves it from the
# Homebrew prefix lib dir. Without this dependency the dylib is
# absent and ORT init fails. See issue #544.
depends_on "onnxruntime"
on_macos do
if Hardware::CPU.arm?
url "https://github.com/yvgude/lean-ctx/releases/download/v${VERSION}/lean-ctx-aarch64-apple-darwin.tar.gz"
sha256 "${AARCH64_DARWIN_SHA}"
else
url "https://github.com/yvgude/lean-ctx/releases/download/v${VERSION}/lean-ctx-x86_64-apple-darwin.tar.gz"
sha256 "${X86_64_DARWIN_SHA}"
end
end
on_linux do
if Hardware::CPU.arm?
url "https://github.com/yvgude/lean-ctx/releases/download/v${VERSION}/lean-ctx-aarch64-unknown-linux-gnu.tar.gz"
sha256 "${AARCH64_LINUX_SHA}"
else
url "https://github.com/yvgude/lean-ctx/releases/download/v${VERSION}/lean-ctx-x86_64-unknown-linux-gnu.tar.gz"
sha256 "${X86_64_LINUX_SHA}"
end
end
def install
bin.install "lean-ctx"
end
test do
assert_match "lean-ctx ${VERSION}", shell_output("#{bin}/lean-ctx --version")
end
end
EOF
sed -i 's/^ //' Formula/lean-ctx.rb
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git add Formula/lean-ctx.rb
git diff --cached --quiet || git commit -m "lean-ctx ${VERSION}"
git push
env:
HOMEBREW_TOKEN: ${{ secrets.HOMEBREW_GITHUB_TOKEN }}
announce-twitter:
name: Announce on Twitter
needs: release
runs-on: ubuntu-latest
if: ${{ !contains(github.ref_name, '-rc') }}
steps:
- uses: actions/checkout@v4 # v4
with:
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: 22
- name: Post release tweet
env:
TWITTER_CONSUMER_KEY: ${{ secrets.TWITTER_CONSUMER_KEY }}
TWITTER_CONSUMER_SECRET: ${{ secrets.TWITTER_CONSUMER_SECRET }}
TWITTER_ACCESS_TOKEN: ${{ secrets.TWITTER_ACCESS_TOKEN }}
TWITTER_ACCESS_SECRET: ${{ secrets.TWITTER_ACCESS_SECRET }}
RELEASE_TAG: ${{ github.ref_name }}
REPO: ${{ github.repository }}
run: node .github/scripts/post-release-tweet.mjs
+171
View File
@@ -0,0 +1,171 @@
name: Security Check
on:
push:
branches: [main]
pull_request:
branches: [main]
permissions:
contents: read
jobs:
security:
name: Security Scan
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Check for dangerous patterns
run: |
echo "## Security Pattern Scan" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
FOUND=0
# Check for unauthorized network libraries
# Allowed: ureq (used for opt-in cloud sync, updates, error reports)
# Allowed: std::net::TcpListener (used for local dashboard server)
# Blocked: reqwest, hyper (heavy HTTP clients not needed)
if grep -rn 'reqwest::' rust/src/ 2>/dev/null; then
echo "::warning::Found reqwest usage — use ureq instead"
echo "- ⚠️ Found reqwest usage (use ureq)" >> $GITHUB_STEP_SUMMARY
FOUND=1
fi
if grep -rn 'hyper::' rust/src/ 2>/dev/null; then
echo "::warning::Found hyper usage — use ureq instead"
echo "- ⚠️ Found hyper usage (use ureq)" >> $GITHUB_STEP_SUMMARY
FOUND=1
fi
# Check for unsafe code
UNSAFE_COUNT=$(grep -rn 'unsafe {' rust/src/ 2>/dev/null | wc -l)
if [ "$UNSAFE_COUNT" -gt 0 ]; then
echo "::warning::Found $UNSAFE_COUNT unsafe blocks"
echo "- ⚠️ Found $UNSAFE_COUNT unsafe blocks" >> $GITHUB_STEP_SUMMARY
grep -rn 'unsafe {' rust/src/ >> $GITHUB_STEP_SUMMARY
FOUND=1
fi
# Check for environment manipulation
if grep -rn '\.env("LD_PRELOAD")' rust/src/ 2>/dev/null; then
echo "::error::Found LD_PRELOAD manipulation — potential library hijacking"
echo "- ❌ Found LD_PRELOAD manipulation" >> $GITHUB_STEP_SUMMARY
FOUND=1
fi
if grep -rn '\.env("DYLD_' rust/src/ 2>/dev/null; then
echo "::error::Found DYLD manipulation — potential library hijacking"
echo "- ❌ Found DYLD manipulation" >> $GITHUB_STEP_SUMMARY
FOUND=1
fi
# Check for hardcoded secrets patterns
if grep -rn 'sk_live_\|sk_test_\|AKIA[0-9A-Z]\|ghp_[a-zA-Z0-9]' rust/src/ 2>/dev/null; then
echo "::error::Found potential hardcoded secrets"
echo "- ❌ Found potential hardcoded secrets" >> $GITHUB_STEP_SUMMARY
FOUND=1
fi
# Check for shell injection vectors
SHELL_INJECT=$(grep -rn 'Command::new("sh")\.arg("-c")\.arg(format!' rust/src/ 2>/dev/null | wc -l)
if [ "$SHELL_INJECT" -gt 0 ]; then
echo "::warning::Found $SHELL_INJECT potential shell injection vectors"
echo "- ⚠️ Found $SHELL_INJECT shell injection patterns" >> $GITHUB_STEP_SUMMARY
FOUND=1
fi
# Check for unwrap() in production code (excluding tests)
UNWRAP_COUNT=$(grep -rn '\.unwrap()' rust/src/ 2>/dev/null | grep -v '#\[test\]' | grep -v 'mod tests' | wc -l)
echo "- ️ Found $UNWRAP_COUNT .unwrap() calls in src/" >> $GITHUB_STEP_SUMMARY
if [ "$FOUND" -eq 0 ]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "✅ No dangerous patterns detected" >> $GITHUB_STEP_SUMMARY
fi
- name: Proprietary code guardrail
run: |
echo "## Proprietary Code Guard" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
LEAK=0
# Keep in sync with .github-ignore. Business/monetization dirs are
# gitignored locally; listing them here is server-side defense-in-depth
# in case the local pre-push hook is not installed.
PRIVATE_PATHS="cloud/ docker-compose.yml .gitlab-ci.yml deploy.sh DEVELOPMENT.md Makefile.deploy docs/business/ memory-bank/ discord-bot/ n8n-workflows/ lab/ server.md"
for path in $PRIVATE_PATHS; do
if [ -e "$path" ]; then
echo "::error::PROPRIETARY CODE DETECTED: $path exists in the GitHub repository!"
echo "- **$path** — must not be on GitHub" >> $GITHUB_STEP_SUMMARY
LEAK=1
fi
done
if [ "$LEAK" -eq 1 ]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These paths belong on GitLab only. See .github-ignore." >> $GITHUB_STEP_SUMMARY
exit 1
fi
echo "No proprietary code found." >> $GITHUB_STEP_SUMMARY
- name: Commercial-plane guard
run: |
echo "## Commercial-plane Guard" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
LEAK=0
# Billing/licensing logic lives in the private lean-ctx-cloud repo
# (oss-plane-separation-v1). core/billing/ is partially OPEN (metering,
# plans, mod), so we name the specific commercial files rather than the
# whole directory — the path guard above can't blanket-block them.
COMMERCIAL_PATHS="rust/src/core/license rust/src/core/licensing/keygen.rs rust/src/cli/license_cmd.rs rust/src/core/billing/success_fee.rs rust/src/core/billing/stripe_invoice.rs docs/contracts/license-v1.md docs/contracts/success-fee-invoice-v1.md"
for path in $COMMERCIAL_PATHS; do
if [ -e "$path" ]; then
echo "::error::COMMERCIAL CODE DETECTED: $path belongs in lean-ctx-cloud, not the open engine!"
echo "- **$path** — implement in lean-ctx-cloud" >> $GITHUB_STEP_SUMMARY
LEAK=1
fi
done
if [ "$LEAK" -eq 1 ]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "The open engine bills nothing and issues no licenses; it only emits the signed savings ledger. See .github-ignore." >> $GITHUB_STEP_SUMMARY
exit 1
fi
echo "No commercial-plane code found." >> $GITHUB_STEP_SUMMARY
- name: Critical files check
if: github.event_name == 'pull_request'
run: |
echo "## Critical Files Modified" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
CRITICAL_FILES="rust/src/shell.rs rust/src/server.rs rust/src/hooks.rs rust/src/core/cache.rs rust/Cargo.toml .github/workflows"
FOUND_CRITICAL=0
for file in $CRITICAL_FILES; do
if git diff --name-only origin/main...HEAD | grep -q "$file"; then
echo "- ⚠️ **$file** modified (requires security review)" >> $GITHUB_STEP_SUMMARY
FOUND_CRITICAL=1
fi
done
if [ "$FOUND_CRITICAL" -eq 0 ]; then
echo "✅ No critical files modified" >> $GITHUB_STEP_SUMMARY
fi
- name: Dependency audit
shell: bash
run: |
set -o pipefail
cargo install cargo-audit
cd rust && cargo audit 2>&1 | tee audit-output.txt
AUDIT_EXIT=${PIPESTATUS[0]}
echo "" >> $GITHUB_STEP_SUMMARY
echo "## Dependency Audit" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
cat audit-output.txt >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
exit $AUDIT_EXIT
+83
View File
@@ -0,0 +1,83 @@
target/
node_modules/
dist/
*.tsbuildinfo
.env
deploy.sh
Dockerfile.web
/src/
/package.json
/package-lock.json
package-lock.json
/tsconfig.json
lean-ctx.config.example.json
INTERNAL_ARCHITECTURE.md
RELEASE_SUMMARY.md
aur/
server.md
.lean-ctx/
.DS_Store
DISCORD_POST.md
DEPLOY_CHECKLIST.md
# ── Discord Bot (private, deployed separately) ──
discord-bot/
# ── Private workflows ──
n8n-workflows/
# ── Lab / Neural (private, not for OSS) ──
lab/
rust/models/
*.onnx
*.pt
# ── Private ──
/cloud/
docker-compose.yml
DEVELOPMENT.md
Makefile.deploy
PROJECT.md
memory-bank/
docs/LEANCTX_CONSOLIDATION_TICKETS.md
deploy_checklist.md
# ── Business / Monetization (private, internal strategy) ──
docs/business/
# ── Local artifacts ──
lean-ctx-chrome-extension.zip
/docs/_local/
test-results/
lean-ctx-report.html
rust/lean-ctx-report.html
*.bak
# ── IDE local config (may contain tokens/env) ──
**/.cursor/
.vscode/mcp.json
.github/mcp.json
*.mcp.json.bak
# Claude Code per-machine config (absolute binary paths, session-local)
**/.claude/settings.local.json*
**/.claude/rules/
# ── Generated / worktrees (local) ──
website/generated/
worktrees/website/generated/
AGENTS.md.lean-ctx.*.bak
docs/gpt/
# ── Internal strategy (never public) ──
/THINKERY-VISION.md
/packages/jetbrains-lean-ctx/build/
# ── Local AI-tooling / scratch dirs (not for OSS) ──
/tmp/
tmp/
markdownai/
.idea/
.serena/
.claude/
+4
View File
@@ -0,0 +1,4 @@
[submodule "aur/lean-ctx-bin"]
path = aur/lean-ctx-bin
url = ssh://aur@aur.archlinux.org/lean-ctx-bin.git
+46
View File
@@ -0,0 +1,46 @@
---
inclusion: always
---
# Context Engineering Layer
The workspace has the `lean-ctx` MCP server installed. You MUST prefer lean-ctx tools over native equivalents for token efficiency and caching.
## Mandatory Tool Preferences
| Use this | Instead of | Why |
|----------|-----------|-----|
| `mcp_lean_ctx_ctx_read` | `readFile`, `readCode` | Cached reads, 10 compression modes, re-reads cost ~13 tokens |
| `mcp_lean_ctx_ctx_multi_read` | `readMultipleFiles` | Batch cached reads in one call |
| `mcp_lean_ctx_ctx_shell` | `executeBash` | Pattern compression for git/npm/test output |
| `mcp_lean_ctx_ctx_search` | `grepSearch` | Compact, .gitignore-aware results |
| `mcp_lean_ctx_ctx_tree` | `listDirectory` | Compact directory maps with file counts |
## When to use native Kiro tools instead
- `fsWrite` / `fsAppend` — always use native (lean-ctx doesn't write files)
- `strReplace` — always use native (precise string replacement)
- `semanticRename` / `smartRelocate` — always use native (IDE integration)
- `getDiagnostics` — always use native (language server diagnostics)
- `deleteFile` — always use native
## Session management
- At the start of a long task, call `mcp_lean_ctx_ctx_preload` with a task description to warm the cache
- Use `mcp_lean_ctx_ctx_compress` periodically in long conversations to checkpoint context
- Use `mcp_lean_ctx_ctx_knowledge` to persist important discoveries across sessions
## Rules
- NEVER loop on edit failures — switch to `mcp_lean_ctx_ctx_edit` immediately
- For large files, use `mcp_lean_ctx_ctx_read` with `mode: "signatures"` or `mode: "map"` first
- For re-reading a file you already read, just call `mcp_lean_ctx_ctx_read` again (cache hit = ~13 tokens)
- When running tests or build commands, use `mcp_lean_ctx_ctx_shell` for compressed output
<!-- lean-ctx-compression -->
OUTPUT STYLE: concise
- Bullet points over paragraphs
- Skip filler words and hedging ("I think", "probably", "it seems")
- 1-sentence explanations max, then code/action
- No repeating what the user said
<!-- /lean-ctx-compression -->
+21
View File
@@ -0,0 +1,21 @@
repos:
- repo: local
hooks:
- id: cargo-fmt
name: cargo fmt
entry: cargo fmt --manifest-path rust/Cargo.toml --check
language: system
types: [rust]
pass_filenames: false
- id: cargo-clippy
name: cargo clippy
entry: cargo clippy --manifest-path rust/Cargo.toml --all-features -- -D warnings
language: system
types: [rust]
pass_filenames: false
- id: cargo-deny
name: cargo deny
entry: cargo deny --manifest-path rust/Cargo.toml check
language: system
types: [rust]
pass_filenames: false
+4
View File
@@ -0,0 +1,4 @@
{
"git.ignoreLimitWarning": true,
"npm.autoDetect": "off"
}
+113
View File
@@ -0,0 +1,113 @@
# Context Engineering Layer
lean-ctx optimizes LLM context by compressing file reads, shell output, and search results.
## Integration Mode: Replace
Native Read/Grep/Glob/Shell are **denied by policy**. lean-ctx MCP tools are the
only path for reading files and running commands:
- **Reads/Search** → `ctx_read`, `ctx_search`, `ctx_compose` (cached, 10 modes)
- **Shell commands** → `ctx_shell` (95+ compression patterns)
- **File editing** → native Edit/StrReplace (lean-ctx only handles READ operations)
- **File finding** → `ctx_glob`, `ctx_tree`
Native tools will return a deny error with instructions to use ctx_* equivalents.
Override: set `hook_mode = "hybrid"` in `~/.config/lean-ctx/config.toml`.
## Subagent Compatibility (Cursor / Claude Code)
Cursor blocks MCP tools in `readonly: true` subagents (Task tool).
lean-ctx tools are MCP tools → they WILL FAIL in readonly subagents.
**Rules for spawning subagents:**
- Always set `readonly: false` when the subagent needs ctx_* tools
- Use `subagent_type: "generalPurpose"` — NOT `"explore"` (always readonly)
- lean-ctx tools declare `readOnlyHint: true` in MCP annotations
**Fallback (MCP unavailable):** deny hook detects MCP-down and allows native tools.
## CLI commands (optimized shell, lower overhead)
```bash
git status # compressed by configured agent shell/wrapper
cargo test # compressed by configured agent shell/wrapper
lean-ctx -c "git status" # only if explicitly requested / documented unwrapped
lean-ctx ls src/ # directory map
```
## Development Workflow
When working on lean-ctx itself:
1. **Before building**: `lean-ctx stop` (LaunchAgent respawns otherwise)
2. **Build**: `cd rust && cargo build --release`
3. **Test**: `cargo test --lib` + `cargo clippy -- -W clippy::all`
4. **Install**: `lean-ctx dev-install` (atomic stop→build→install→restart)
## Session Continuity
lean-ctx automatically persists session context across restarts:
- **Findings**: Recent tool results (reads, searches, test outcomes)
- **Decisions**: Architecture choices made during the session
- **Files**: Touched files with summaries and modification status
- **Progress**: Task completion state and next steps
This data is delivered through the first tool call's `--- AUTO CONTEXT ---`
briefing (default `minimal_overhead = true`: initialize instructions stay
byte-stable for provider prompt caching, #498). With `minimal_overhead = false`
it is additionally injected at session start via the `ACTIVE SESSION` LITM block.
### Active Documentation (Agent Responsibility)
After completing a significant task (implementation, bugfix, refactoring):
1. Record the decision: `ctx_knowledge(action="remember", category="decision", content="...")`
2. Record progress: `ctx_session(action="task", value="<current task> [N%]")`
3. Record blockers: `ctx_knowledge(action="remember", category="blocker", content="...")`
After 30+ tool calls without documentation:
- lean-ctx will prompt with `[CHECKPOINT: please document current progress]`
- Respond by calling `ctx_session(action="task")` with current status
## Provider Pipeline (Context Engine)
External data sources (GitHub, GitLab, Jira, Postgres, MCP bridges, custom REST) are first-class citizens.
All provider data flows through the same consolidation pipeline:
1. `ContextProvider::execute()` → raw `ProviderResult`
2. `consolidation::consolidate()``ConsolidationArtifacts` (BM25 chunks, graph edges, knowledge facts, cache entries)
3. `apply_artifacts_to_stores()` → persists to BM25 index, Graph index, ProjectKnowledge, Session cache (background thread)
This means `ctx_semantic_search` finds issues/PRs/tickets, `ctx_knowledge` recalls provider facts,
and `ctx_read` shows cross-source hints (e.g. "Issue #42 references this file").
## Quality Bar
- Zero clippy warnings, all tests pass
- Security: PathJail, Shell Allowlist, bounded_lock, no hardcoded secrets
- No mock data, no placeholders, no stubs
## Output Determinism (#498)
Tool outputs MUST be deterministic functions of (file content, mode, CRP mode, task).
Provider-side prompt caching (Anthropic 90%, OpenAI 50% discount) rewards byte-stable text;
any timestamp, counter or random element in tool output bodies defeats it.
- No timestamps/counters in output bodies. Artifact paths are content-addressed
(see `save_tee`: `{cmd_slug}_{blake3(cmd)[..8]}.log`).
- Dynamic additions (hints, checkpoints) only as state-triggered suffixes with stable headers.
- Regression guard: determinism tests in `ctx_read/tests.rs`, `ctx_search.rs`, `shell/redact.rs`.
<!-- lean-ctx -->
## lean-ctx
lean-ctx is active — the MCP tools replace native equivalents.
Full rules: LEAN-CTX.md (open on demand — do not auto-load).
<!-- /lean-ctx -->
<!-- lean-ctx-compression -->
OUTPUT STYLE: concise
- Bullet points over paragraphs
- Skip filler words and hedging ("I think", "probably", "it seems")
- 1-sentence explanations max, then code/action
- No repeating what the user said
<!-- /lean-ctx-compression -->
+1202
View File
File diff suppressed because it is too large Load Diff
+138
View File
@@ -0,0 +1,138 @@
# lean-ctx Benchmark: Head-to-Head Comparison
> Generated by lean-ctx v3.8.18 on 2026-07-02 07:06:37
**Project:** `.`
**Files measured:** 50
**Total raw tokens:** 533.2K
## Methodology
All lean-ctx measurements are **real values** measured on the test repository. Competitor numbers use **published figures** from their official documentation, papers, or README files. Sources are cited in the comparison table.
- **Token counting**: tiktoken `o200k_base` (GPT-4o tokenizer)
- **Compression**: Each lean-ctx read mode is applied to the same files
- **Latency**: Wall-clock time, median of all measured files
- **Quality**: Preservation score (structural + semantic fidelity)
## Compression Comparison
| Tool | Compression | Tokens | Source |
|------|------------:|-------:|--------|
| Raw file read | 0% | 533.2K | Baseline — no compression applied |
| Repomix | 70% | 160.0K | Repomix docs (Tree-sitter compress mode) |
| aider /map | 85% | 80.0K | aider docs (repo-map with ctags/tree-sitter) |
| codebase-memory-mcp | 99% | 4.3K | arXiv paper (graph-query extraction only) |
| TokenForge | 70% | 160.0K | TokenForge README (tree-sitter code folding 40-70%; full-stack: code/command/conversation/json/mcp-schema) |
| **lean-ctx (map)** | **98.1%** | **8.0K** | Measured |
| **lean-ctx (signatures)** | **96.7%** | **14.0K** | Measured |
| **lean-ctx (aggressive)** | **16.5%** | **449.7K** | Measured |
| **lean-ctx (entropy)** | **0.4%** | **531.4K** | Measured |
## lean-ctx Mode Performance
| Mode | Compression | Latency | Quality | Use Case |
|------|------------:|--------:|--------:|----------|
| full | 0.0% | 0μs | 100% | Editing files (cached, ~13 tok on re-read) |
| map | 98.1% | 8.2ms | 78% | Understanding structure, deps, exports |
| signatures | 96.7% | 2.6ms | 96% | API surface only |
| aggressive | 16.5% | 344μs | 100% | Maximum compression for large files |
| entropy | 0.4% | 8.4ms | 100% | Information-theoretic filtering |
## Search Latency
| Query | BM25 Latency | Results |
|-------|-------------:|--------:|
| `function` | 456μs | 10 |
| `error handling` | 982μs | 10 |
| `configuration` | 60μs | 10 |
| `parse` | 415μs | 10 |
| `test` | 671μs | 10 |
| **Average** | **516μs** | — |
## Cold Start Performance
| Phase | Duration |
|-------|--------:|
| File scan | 220μs |
| BM25 index build | 672.5ms |
| First file read + tokenize | 240μs |
| **Total cold start** | **673.0ms** |
## Disk Footprint
| Component | Size |
|-----------|-----:|
| BM25 index | 6.7 MB |
| Total `.lean-ctx/` | 6.7 MB |
## Feature Comparison
| Feature | Raw | Repomix | aider | codebase-memory | **lean-ctx** |
|---------|:---:|:-------:|:-----:|:---------------:|:------------:|
| Multi-mode compression | — | — | — | — | ✅ |
| BM25 code search | — | — | — | ✅ | ✅ |
| Session caching | — | — | ✅ | ✅ | ✅ |
| Cross-session memory (CCP) | — | — | — | ✅ | ✅ |
| Shell output compression | — | — | — | — | ✅ |
| Call graph analysis | — | — | — | — | ✅ |
| Repo map generation | — | ✅ | ✅ | — | ✅ |
| Knowledge base | — | — | — | ✅ | ✅ |
| Tree-sitter AST (26 langs) | — | ✅ | ✅ | — | ✅ |
| MCP server | — | — | — | ✅ | ✅ |
**lean-ctx feature count:** 23 operations across 81 MCP tools
## Session Simulation (30-min coding)
| Approach | Tokens | Cost | Savings |
|----------|-------:|-----:|--------:|
| Raw (no compression) | 721.9K | $1.805 | — |
| lean-ctx (no CCP) | 90.7K | $0.227 | 87.4% |
| **lean-ctx + CCP** | **85.6K** | **$0.214** | **88.1%** |
## Compression Visualization
```
Compression % (higher = better)
Raw file read 0.0%
lean-ctx (entropy) 0.4%
lean-ctx (aggressive) ████████ 16.5%
Repomix ███████████████████████████████████ 70.0%
TokenForge ███████████████████████████████████ 70.0%
aider /map ██████████████████████████████████████████ 85.0%
lean-ctx (signatures) ████████████████████████████████████████████████ 96.7%
lean-ctx (map) █████████████████████████████████████████████████ 98.1%
codebase-memory-mcp █████████████████████████████████████████████████ 99.2%
```
## System Information
- **OS:** macos aarch64
- **CPU:** Apple M5 (10 cores)
- **RAM:** 24.0 GB
- **lean-ctx:** v3.8.18
- **Rust:** rustc 1.96.1 (31fca3adb 2026-06-26)
## Reproducibility
```bash
# Install lean-ctx
cargo install lean-ctx
# Run the comparative benchmark on this repo
lean-ctx benchmark compare
# Run on a specific repository
lean-ctx benchmark compare --repo /path/to/repo
# Output to file
lean-ctx benchmark compare --output BENCHMARKS.md
```
---
*Generated by [lean-ctx](https://leanctx.com) v3.8.18 — Context Runtime for AI Agents*
**Disclaimer:** Competitor numbers are from published sources (docs, papers, READMEs). lean-ctx numbers are measured live. Different test repos will produce different results. Run `lean-ctx benchmark compare` on your own codebase for project-specific numbers.
+7953
View File
File diff suppressed because it is too large Load Diff
+119
View File
@@ -0,0 +1,119 @@
# Contributor License Agreement
Thank you for your interest in contributing to **lean-ctx** ("the Project"),
maintained by Yves Gugger ("the Maintainer"). This Contributor License Agreement
("Agreement") clarifies the intellectual-property license granted with
Contributions from any person or entity. It protects both You and the Project,
and it does **not** change Your right to use Your own Contributions for any other
purpose.
By making a Contribution to the Project, You accept and agree to the following
terms for Your present and future Contributions. Except for the licenses granted
herein, You retain all right, title, and interest in and to Your Contributions.
## 1. Definitions
**"You"** (or **"Your"**) means the individual or legal entity making a
Contribution, or the legal entity on whose behalf the individual is authorized to
act.
**"Contribution"** means any original work of authorship, including any
modifications or additions to an existing work, that is intentionally submitted by
You to the Project for inclusion in, or documentation of, any of the products
owned or managed by the Project. "Submitted" means any form of electronic, verbal,
or written communication sent to the Project or its representatives — including but
not limited to communication on pull requests, issues, and electronic mailing
lists — but excluding communication that is conspicuously marked, or otherwise
designated in writing by You, as "Not a Contribution."
## 2. Grant of Copyright License
Subject to the terms and conditions of this Agreement, You hereby grant to the
Maintainer and to recipients of software distributed by the Project a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to
reproduce, prepare derivative works of, publicly display, publicly perform,
sublicense, and distribute Your Contributions and such derivative works. This
license **includes the right to relicense** the Contributions, in whole or in
part, under any license terms — including proprietary and dual-license terms — as
the Maintainer may determine.
## 3. Grant of Patent License
Subject to the terms and conditions of this Agreement, You hereby grant to the
Maintainer and to recipients of software distributed by the Project a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in
this section) patent license to make, have made, use, offer to sell, sell, import,
and otherwise transfer the Work, where such license applies only to those patent
claims licensable by You that are necessarily infringed by Your Contribution(s)
alone or by combination of Your Contribution(s) with the Work to which such
Contribution(s) was submitted. If any entity institutes patent litigation against
You or any other entity (including a cross-claim or counterclaim in a lawsuit)
alleging that Your Contribution, or the Work to which You have contributed,
constitutes direct or contributory patent infringement, then any patent licenses
granted to that entity under this Agreement for that Contribution or Work shall
terminate as of the date such litigation is filed.
## 4. Representations
You represent that:
1. You are legally entitled to grant the above licenses. If Your employer(s) has
rights to intellectual property that You create that includes Your
Contributions, You represent that You have received permission to make
Contributions on behalf of that employer, that Your employer has waived such
rights for Your Contributions to the Project, or that Your employer has executed
a separate Corporate CLA with the Project.
2. Each of Your Contributions is Your original creation (see Section 6 for
submissions on behalf of others).
3. Your Contribution submissions include complete details of any third-party
license or other restriction (including, but not limited to, related patents and
trademarks) of which You are personally aware and which are associated with any
part of Your Contributions.
## 5. No Obligation
You understand that the decision to include Your Contribution in any project or
source repository is entirely that of the Maintainer, and this Agreement does not
guarantee that the Contributions will be included in any product.
## 6. Third-Party Works
Should You wish to submit work that is not Your original creation, You may submit it
to the Project separately from any Contribution, identifying the complete details of
its source and of any license or other restriction (including, but not limited to,
related patents, trademarks, and license agreements) of which You are personally
aware, and conspicuously marking the work as "Submitted on behalf of a third-party:
[named here]".
## 7. Maintenance of Representations
You agree to notify the Project of any facts or circumstances of which You become
aware that would make these representations inaccurate in any respect.
## 8. Relationship to the Open-Source License
Contributions to the Project remain licensed to the public under the Project's
then-current open-source license (the Apache License 2.0). The additional rights
granted to the Maintainer in Sections 2 and 3 exist **alongside**, and do not
diminish, that public license. The free and open-source distribution of lean-ctx
for individual developers is and remains the Project's commitment.
---
## How to sign
You only need to sign **once**. When you open your first pull request, the CLA
Assistant bot will comment with instructions and a status check. To sign, post the
following as a comment on the pull request:
> I have read the CLA Document and I hereby sign the CLA
Your signature (your GitHub username and the signing pull request) is recorded in
this repository under the `cla-signatures` branch. Future pull requests are then
accepted without re-signing.
---
> _This document is adapted from the widely used Apache Individual Contributor
> License Agreement and the CLA Assistant template. It is provided in good faith;
> for commercial reliance it should be reviewed by legal counsel._
+77
View File
@@ -0,0 +1,77 @@
# Code of Conduct
This project follows the Contributor Covenant Code of Conduct.
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our community include:
- Demonstrating empathy and kindness toward other people
- Being respectful of differing opinions, viewpoints, and experiences
- Giving and gracefully accepting constructive feedback
- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience
- Focusing on what is best not just for us as individuals, but for the overall community
Examples of unacceptable behavior include:
- The use of sexualized language or imagery, and sexual attention or advances of any kind
- Trolling, insulting or derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or email address, without their explicit permission
- Other conduct which could reasonably be considered inappropriate in a professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official email address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at **yves@pounce.ch**. All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of actions.
**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1.
Community Impact Guidelines were inspired by the [Mozilla Code of Conduct Enforcement Ladder](https://github.com/mozilla/diversity).
+315
View File
@@ -0,0 +1,315 @@
# LeanCTX Protocol Family & Contracts (v1)
LeanCTX is infrastructure. Contracts are the stable promises that client integrations, CI gates, proof artifacts, and future plugins rely on.
## Architecture positioning
```
MCP = how agents call LeanCTX (external interoperability)
LCP = how LeanCTX understands, transforms, and governs context (internal semantics)
```
MCP is the transport. The contracts below define what flows through it.
## Versioning rules (SemVer policy)
- **Schema versions are integers** (`schema_version` / `contract_version`).
- **Breaking change** => bump the corresponding version and add migration notes.
- Examples: removing fields, changing field types, changing required fields, changing error semantics/status codes.
- **Non-breaking change** => keep version, document additive changes.
- Examples: adding optional fields, adding new tools, adding new docs pages.
- **Compatibility**:
- Newer runtimes should be able to **read older artifacts** where possible (at least for proofs / observability).
- If multiple versions are supported concurrently, support is **explicitly documented**.
### Release SemVer mapping
For the `lean-ctx` release version (`MAJOR.MINOR.PATCH`):
| Change | Release bump | Contract effect |
|---|---|---|
| Bugfix, perf, docs, new compression pattern | PATCH | none |
| New tool, new endpoint, new optional field | MINOR | additive — contract version unchanged |
| Breaking change to a `stable` contract | MAJOR | contract version bump (`v1``v2`) |
| Any change to a `frozen` contract doc | forbidden | publish a **new** `-v2.md` file; the `-v1.md` file stays immutable |
### Contract file rule (v1 → v2)
A versioned contract doc (`docs/contracts/<name>-vN.md`) is an **artifact, not a living document**:
- Frozen docs never change — CI (`rust/tests/contracts_frozen.rs`) hashes them and fails on any edit.
- A semantic revision lands as a **new file** (`<name>-v2.md`); the old file remains for existing integrations and gains a deprecation pointer in CONTRACTS.md (not in the frozen file itself).
- Typo fixes in frozen docs are deliberately treated as changes: regenerate the hash snapshot via `LEANCTX_UPDATE_FROZEN_HASHES=1 cargo test --test contracts_frozen` and justify it in the PR.
### Deprecation policy
- A surface (CLI command, MCP tool, HTTP endpoint, config key, contract version) is deprecated **at least 2 minor releases** before removal.
- Every deprecation is recorded in [`DEPRECATIONS.toml`](rust/data/DEPRECATIONS.toml) (compiled into the binary; lives inside `rust/` so `cargo publish` can package it) with `announced_in`, `earliest_removal`, and a `replacement`.
- `lean-ctx doctor` warns about every active deprecation shipping in the installed build.
- Every release that announces or executes a removal lists it in a dedicated **Deprecations** section of the CHANGELOG.
- `experimental` contracts are exempt — they may change or disappear without notice.
## Stability matrix
Status of every contract document (SSOT: `rust/src/core/contracts.rs::contract_docs()`; enforced by `rust/tests/contracts_frozen.rs` — no doc may stay unclassified):
| Status | Meaning |
|---|---|
| `frozen` | Normative surface immutable; change = new `-v2.md` file. CI-enforced via content hash. |
| `stable` | Additive evolution allowed; breaking change requires version bump + migration notes. |
| `experimental` | May change or disappear without notice. |
| Contract | Doc | Version | Status |
|---|---|---|---|
| HTTP MCP | `docs/contracts/http-mcp-contract-v1.md` | 1 | frozen |
| Team Server | `docs/contracts/team-server-contract-v1.md` | 1 | frozen |
| Context IR | `docs/contracts/context-ir-v1.md` | 1 | frozen |
| Local-Free Invariant | `docs/contracts/local-free-invariant-v1.md` | 1 | frozen |
| OSS Plane Separation | `docs/contracts/oss-plane-separation-v1.md` | 1 | frozen |
| Billing Plane | `docs/contracts/billing-plane-v1.md` | 1 | frozen |
| WASM ABI | `docs/contracts/wasm-abi-v1.md` | 1 | frozen |
| Capabilities | `docs/contracts/capabilities-contract-v1.md` | 1 | stable¹ |
| Billing Plane v2 | `docs/contracts/billing-plane-v2.md` | 2 | stable |
| A2A | `docs/contracts/a2a-contract-v1.md` | 1 | stable |
| Attention Layout Driver | `docs/contracts/attention-layout-driver-v1.md` | 1 | stable |
| Autonomy Drivers | `docs/contracts/autonomy-drivers-v1.md` | 1 | stable |
| CCP Session Bundle | `docs/contracts/ccp-session-bundle-v1.md` | 1 | stable |
| Conformance | `docs/contracts/conformance-v1.md` | 1 | stable |
| Degradation Policy | `docs/contracts/degradation-policy-v1.md` | 1 | stable |
| Extension Trust | `docs/contracts/extension-trust-v1.md` | 1 | stable |
| Extractors | `docs/contracts/extractors-v1.md` | 1 | stable |
| Gotchas/Reminders | `docs/contracts/gotchas-reminders-contract-v1.md` | 1 | stable |
| Graph Reproducibility | `docs/contracts/graph-reproducibility-contract-v1.md` | 1 | stable |
| Handoff Transfer Bundle | `docs/contracts/handoff-transfer-bundle-v1.md` | 1 | stable |
| Intent Route | `docs/contracts/intent-route-v1.md` | 1 | stable |
| Knowledge Policy | `docs/contracts/knowledge-policy-contract-v1.md` | 1 | stable |
| Memory Boundary | `docs/contracts/memory-boundary-contract-v1.md` | 1 | stable |
| Persona Spec | `docs/contracts/persona-spec-v1.md` | 1 | stable |
| Provider Framework | `docs/contracts/provider-framework-contract-v1.md` | 1 | stable |
| Tokenizer Translation Driver | `docs/contracts/tokenizer-translation-driver-v1.md` | 1 | stable |
| Workflow Evidence Ledger | `docs/contracts/workflow-evidence-ledger-v1.md` | 1 | stable |
| Wrapped Permalink | `docs/contracts/wrapped-permalink-v1.md` | 1 | stable |
| Hosted Personal Index | `docs/contracts/hosted-personal-index-v1.md` | 1 | experimental |
| Personal Cloud Encryption | `docs/contracts/personal-cloud-encryption-v1.md` | 1 | experimental |
| Context Snapshot | `docs/contracts/context-snapshot-v1.md` | 1 | experimental |
¹ The capabilities document is additive **by design**: its drift test binds the doc's key list to `server_capabilities::TOP_LEVEL_KEYS`, so the doc must grow whenever a key is added. Freezing the file would contradict its own contract; removal or mutation of existing keys remains a breaking change.
Clients can read this matrix at runtime: `GET /v1/capabilities` returns a `contract_status` map (contract-id → status) next to the existing `contracts` version map.
The OpenAPI document (`GET /v1/openapi.json`) is part of the frozen surface: `rust/tests/openapi_stability.rs` compares the endpoint inventory against `docs/reference/openapi-v1.snapshot.json` — additive diffs are allowed, removed or mutated routes fail CI.
## Current contract versions (SSOT, machine-checked)
<!-- leanctx-contracts-kv:begin -->
leanctx.contract.mcp_manifest.schema_version=1
leanctx.contract.context_proof_v1.schema_version=1
leanctx.contract.context_ir_v1.schema_version=1
leanctx.contract.intent_route_v1.schema_version=1
leanctx.contract.degradation_policy_v1.schema_version=1
leanctx.contract.workflow_evidence_ledger_v1.schema_version=1
leanctx.contract.autonomy_drivers_v1.schema_version=1
leanctx.contract.tokenizer_translation_driver_v1.schema_version=1
leanctx.contract.attention_layout_driver_v1.schema_version=1
leanctx.contract.verification_observability_v1.schema_version=1
leanctx.contract.handoff_ledger_v1.schema_version=1
leanctx.contract.handoff_transfer_bundle_v1.schema_version=1
leanctx.contract.ccp_session_bundle_v1.schema_version=1
leanctx.contract.knowledge_policy_v1.schema_version=1
leanctx.contract.graph_reproducibility_v1.schema_version=1
leanctx.contract.a2a_snapshot_v1.schema_version=1
leanctx.contract.memory_boundary_v1.schema_version=1
leanctx.contract.gotchas_reminders_v1.schema_version=1
leanctx.contract.provider_framework_v1.schema_version=1
leanctx.contract.http_mcp.contract_version=1
leanctx.contract.team_server.contract_version=1
leanctx.contract.context_snapshot_v1.schema_version=1
<!-- leanctx-contracts-kv:end -->
---
## Core Context Contracts
Foundational representations that all other contracts build upon.
### Context IR v1 (Intermediate Representation)
The canonical representation of all context flowing through LeanCTX. Every tool call is recorded with source, lineage, tokens, compression ratio, and safety metadata.
- **Doc**: `docs/contracts/context-ir-v1.md`
- **Runtime source**: `rust/src/core/context_ir.rs`
- **Surface**: Recorded in hot-path after every `call_tool`; exported via `ctx_proof`; persisted to `~/.lean-ctx/context_ir_v1.json`
### Context Proof v1 (Verification Artifacts)
Cryptographic proofs that document what context was produced, how it was compressed, and whether it's reproducible.
- **Runtime source**: `rust/src/core/context_proof.rs`
- **Surface**: `ctx_proof` tool; exports to `project/.lean-ctx/proofs/`
### Verification Observability v1
Runtime observability for output verification (compression safety checks).
- **Runtime source**: `rust/src/core/verification_observability.rs`
- **Surface**: Verify footer in tool outputs when profile-enabled
### Context Snapshot v1 (Context Time Machine)
A git-anchored, signed, point-in-time record of the context-layer state — lineage (from IR), ledger Φ-scores + item states, token ROI, and session slice. Snapshots chain into an append-only timeline you can rewind, reproduce, resume, or share. Distilled-by-default (never raw transcripts), content-addressed id (BLAKE3), ed25519-signed.
- **Doc**: `docs/contracts/context-snapshot-v1.md`
- **Runtime source**: `rust/src/core/context_snapshot/`
- **Surface**: Phase 0 contract (#1023); builder + `snapshot`/`timeline` CLI land in Phase 1 (#1024)
---
## Runtime Contracts
Govern how the runtime processes, budgets, and degrades context.
### Degradation Policy v1 (Budgets/SLOs)
- **Doc**: `docs/contracts/degradation-policy-v1.md`
- **Runtime source**: `rust/src/core/degradation_policy.rs`
- **Surface**: Enforced at tool-call boundary when enabled
### Workflow Evidence Ledger v1
- **Doc**: `docs/contracts/workflow-evidence-ledger-v1.md`
- **Runtime source**: `rust/src/core/evidence_ledger.rs`
- **Surface**: `ctx_workflow` evidence-gated transitions + automatic tool receipts
### Autonomy Drivers v1
- **Doc**: `docs/contracts/autonomy-drivers-v1.md`
- **Runtime source**: `rust/src/core/autonomy_drivers.rs` + `rust/src/tools/autonomy.rs`
- **Surface**: Deterministic driver planner + bounded driver reports; proof export via `ctx_proof`
### Intent Route v1 (Orchestration Routing)
- **Doc**: `docs/contracts/intent-route-v1.md`
- **Runtime source**: `rust/src/core/intent_router.rs`
- **Surface**: `ctx_intent` with `format=json` returns `IntentRouteV1`
### Tokenizer-aware Translation Driver v1
- **Doc**: `docs/contracts/tokenizer-translation-driver-v1.md`
- **Runtime source**: `rust/src/core/tokenizer_translation_driver.rs`
- **Surface**: Deterministic ruleset selection (model_key -> ruleset) + bounded translation
### Attention-aware Layout Driver v1
- **Doc**: `docs/contracts/attention-layout-driver-v1.md`
- **Runtime source**: `rust/src/core/attention_layout_driver.rs`
- **Surface**: Deterministic reorder for delivery surfaces when profile-enabled
---
## Memory & Collaboration Contracts
Define how context persists, transfers between agents, and crosses boundaries.
### CCP Session Bundle v1
- **Doc**: `docs/contracts/ccp-session-bundle-v1.md`
- **Runtime source**: `rust/src/core/ccp_session_bundle.rs` + `rust/src/core/session.rs`
- **Surface**: `ctx_session action=export|import` (redacted-by-default, bounded, replayable)
### Knowledge Policy v1
- **Doc**: `docs/contracts/knowledge-policy-contract-v1.md`
- **Runtime source**: `rust/src/core/memory_policy.rs` + `rust/src/core/knowledge.rs`
- **Surface**: `ctx_knowledge action=policy value=show|validate`
### Graph Reproducibility v1
- **Doc**: `docs/contracts/graph-reproducibility-contract-v1.md`
- **Runtime source**: `rust/src/core/property_graph/*`
- **Surface**: `ctx_impact` / `ctx_architecture` with `format=json`
### A2A Contract v1 (Multi-Agent)
- **Doc**: `docs/contracts/a2a-contract-v1.md`
- **Runtime source**: `rust/src/core/agents.rs` + `rust/src/core/a2a/*`
- **Surface**: `ctx_agent`, `ctx_task`, rate limiting, cost attribution
### Handoff Transfer Bundle v1
- **Doc**: `docs/contracts/handoff-transfer-bundle-v1.md`
- **Runtime source**: `rust/src/core/handoff_transfer_bundle.rs`
- **Surface**: `ctx_handoff action=export|import` (redacted-by-default, bounded, identity-aware)
### Memory Boundary v1
- **Doc**: `docs/contracts/memory-boundary-contract-v1.md`
- **Runtime source**: `rust/src/core/memory_boundary.rs`
- **Surface**: `FactPrivacy` scoping, cross-project gates, audit events
### Gotchas/Reminders v1
- **Doc**: `docs/contracts/gotchas-reminders-contract-v1.md`
- **Runtime source**: `rust/src/core/gotcha_tracker/model.rs`
- **Surface**: Time-bounded reminders with provenance and decay
---
## Extension Contracts
Interfaces for external integrations and future plugin system.
### Provider Framework v1 (Context I/O)
- **Doc**: `docs/contracts/provider-framework-contract-v1.md`
- **Runtime source**: `rust/src/core/providers/` + `rust/src/tools/ctx_provider.rs`
- **Surface**: `ctx_provider` tool (GitLab issues, MRs, pipelines); TTL-based cache; redaction on all outputs
- **Future**: This contract defines the shape for third-party Context Provider plugins
### CompressionPattern (planned v1)
- **Status**: Interface extracted from `rust/src/core/patterns/mod.rs`
- **Future**: Plugin-loadable compression patterns for proprietary CLI tools
### LcpTool (planned v1)
- **Status**: Mirrors existing `McpTool` trait in `rust/src/server/tool_trait.rs`
- **Future**: Plugin-registered tools that inherit the full runtime pipeline
---
## Transport Contracts
Define how LeanCTX communicates with the outside world.
### MCP Manifest v1 (Tool Inventory)
- **Artifact**: `website/generated/mcp-tools.json`
- **Schema**: `schema_version` + normalized tool entries (`name`, `description`, `input_schema`, `schema_md5`)
- **Runtime source**: `rust/src/core/mcp_manifest.rs`
### HTTP MCP v1
- **Doc**: `docs/contracts/http-mcp-contract-v1.md`
- **Stable endpoints**: `/health`, `/v1/manifest`, `/v1/tools`, `/v1/tools/call`, `/v1/events`, `/v1/context/summary`
- **Event schema**: `ContextEventV1` with `version`, `parentId`, `consistencyLevel`
- **Typed errors**: JSON `error_code` + `error`
### Team Server v1
- **Doc**: `docs/contracts/team-server-contract-v1.md`
- **Workspaces**: `x-leanctx-workspace` header + `workspaceId` body + deterministic fallback
- **Audit log**: JSONL with `argumentsMd5` only (no raw args)
---
## Compatibility matrix
| Integration | Transport | Contracts relied on | Setup |
|---|---|---|---|
| Cursor | MCP (stdio) + Shell Hook | MCP manifest v1 + tool schemas + shell patterns | `lean-ctx setup` |
| Claude Code | MCP (stdio) + Shell Hook | MCP manifest v1 + tool schemas + shell patterns | `lean-ctx init --agent claude` |
| CodeBuddy | MCP (stdio) + Shell Hook | MCP manifest v1 + tool schemas + shell patterns | `lean-ctx init --agent codebuddy` |
| GitHub Copilot | MCP (stdio) + Shell Hook | MCP manifest v1 + tool schemas | `lean-ctx init --agent copilot` |
| Remote agents | HTTP | HTTP MCP v1 + typed errors | `lean-ctx serve` |
| Teams | HTTP | Team Server v1 + audit log | `lean-ctx team serve` |
| Future plugins | In-process / Subprocess | Provider v1 + CompressionPattern v1 + LcpTool v1 | `~/.config/lean-ctx/plugins/` |
+261
View File
@@ -0,0 +1,261 @@
# Contributing to lean-ctx
Thanks for your interest in lean-ctx — contributions are welcome.
## Spec-driven workflow (non-trivial changes)
lean-ctx develops non-trivial features **spec-anchored** (review-gated SDD):
```text
spec → plan → tasks → implement (impact-first) → verify → evidence
```
1. **Spec**`specs/NNN-<slug>/spec.md` (copy `specs/_template/`); `NNN` = tracking
issue iid, acceptance criteria in EARS. One feature = one dir.
2. **Plan** — draft in your agent's plan mode (e.g. Cursor Plan Mode `Shift+Tab`),
then distill the approved approach into `specs/NNN-<slug>/plan.md`.
Review before coding.
3. **Tasks**`specs/NNN-<slug>/tasks.md`: atomic, individually testable.
4. **Implement** — impact-first: run `ctx_impact` (or `lean-ctx graph impact <file>`)
before editing `rust/src/**` and verify the affected tests.
5. **Verify**`scripts/preflight.sh fast` + the affected tests.
6. **Evidence** — cite the spec in commits (`refs specs/NNN-<slug>`), link issue #NNN.
Skip the full loop for trivial fixes; use it for features, contracts, and anything
touching the tool/CLI surface. See `specs/README.md`.
## Quick start (core Rust binary)
### Prerequisites
- Rust (stable) via [rustup](https://rustup.rs/)
- Git
- A C toolchain (`cc`, plus `cmake` for `aws-lc`) — several dependencies
(jemalloc, `aws-lc`, …) build from source
### Setup
```bash
git clone https://github.com/yvgude/lean-ctx.git
cd lean-ctx/rust
cargo build
cargo test
```
### Quality bar (required)
```bash
cargo fmt --check
cargo clippy --all-targets --all-features -- -D warnings
cargo test --all-features
cargo test --release
```
### Pre-push gate (CI parity)
`make setup-hooks` wires a pre-push hook that runs `scripts/preflight.sh fast`
the deterministic CI jobs (fmt, clippy, rustdoc, generated-docs drift, Windows
cross-compile) mirrored locally so you catch them in seconds, not after a 50-min
matrix. It is **change-aware**: a docs-only push (README, CHANGELOG, `*.md`,
website, …) skips the Rust gates entirely, while CI still runs every job as the
source of truth. Run the full gate (everything + `cargo test --lib`, ignoring the
diff) with `make preflight`. Bypass once with `SKIP_PREFLIGHT=1 git push`.
A change to contract code (`proxy/`, `tools/`, `config/schema/`) that ships no
test signal triggers a **no-test advisory**; export
`LEAN_CTX_PREFLIGHT_STRICT_TESTS=1` to make it blocking.
## Building across worktrees & disk usage
lean-ctx pulls in a **heavy native-dependency tree** (jemalloc, an `aws-lc`
crypto build, tree-sitter grammars, …), so a debug build is larger than the Rust
source alone suggests. A couple of things worth knowing so it doesn't surprise
your disk:
- **Each `git worktree` gets its own `target/`.** Keep several PR checkouts open
and Cargo compiles the full native tree *per worktree*, sharing nothing
between them.
- **`target/debug` never garbage-collects.** Stale incremental units and old
dependency versions accumulate, so one heavily-rebuilt `target/` can reach
**tens of GB** (vs. ~2 GB for a clean build).
### A shared compilation cache (recommended)
[`sccache`](https://github.com/mozilla/sccache) deduplicates dependency compiles
across worktrees and branches, without the build-lock contention a shared
`CARGO_TARGET_DIR` introduces:
```bash
cargo install sccache
export RUSTC_WRAPPER=sccache # add to your shell profile
```
> A single shared `CARGO_TARGET_DIR` also dedups, but Cargo holds a per-target
> build lock, so concurrent builds across worktrees **serialize**.
### Prune stale artifacts
[`cargo-sweep`](https://github.com/holmgr/cargo-sweep) drops build artifacts past
a cutoff so `target/` can't grow without bound:
```bash
cargo install cargo-sweep
cargo sweep --time 7 # remove artifacts unused for > 7 days
```
### Reclaim space fast
`target/` is always safe to delete — it's pure build output and regenerates on
the next build:
```bash
cargo clean # this checkout's target/
du -sh target # check current size
```
Debug info is the bulk of that size: this repo sets
`[profile.dev] debug = "line-tables-only"`, which keeps `file:line` in panics and
backtraces while dropping full variable-level data. Set `debug = 2` in a local
profile override if you need to step-debug.
## Cookbook / SDK / extensions (optional)
If you contribute to `cookbook/` or `packages/`, youll also need:
- Node.js (>= 22.12.0)
- npm
```bash
cd cookbook
npm ci
npm test
```
## Repo structure
```text
lean-ctx/
├─ rust/ # core binary (CLI + MCP server + shell hook)
│ ├─ src/
│ │ ├─ main.rs # CLI entry point
│ │ ├─ lib.rs # library entry point (shared core)
│ │ ├─ mcp_stdio.rs # MCP stdio transport
│ │ ├─ server/ # MCP server state + dispatch
│ │ ├─ tools/ # MCP tool handlers (ctx_read, ctx_shell, ...)
│ │ ├─ core/ # cache, compression, patterns/, memory, graphs, ...
│ │ ├─ cli/ # CLI subcommands (setup, init, read, ...)
│ │ └─ hooks/ # editor/agent installers (Cursor, Claude Code, ...)
│ └─ tests/ # integration/e2e/adversarial tests
├─ cookbook/ # real examples + lean-ctx-client
├─ packages/ # editor integrations (VSCode, Chrome, JetBrains, ...)
├─ docs/ # repo docs (developer-facing)
└─ website/generated/ # generated schemas (tool + TDD schema)
```
## Common contribution types
### Add a shell compression pattern
1. Add a new module in `rust/src/core/patterns/<tool>.rs`
2. Implement:
```rust
pub fn compress(command: &str, output: &str) -> Option<String>
```
3. Register the module + routing in `rust/src/core/patterns/mod.rs` (`try_specific_pattern`)
4. Add tests (unit tests in the module or integration tests in `rust/tests/`)
5. Run the quality checks above
Tip: open a ticket via the [New Compression Pattern](.github/ISSUE_TEMPLATE/compression_pattern.md) template and include raw output + expected compressed output.
### Add or update an MCP tool
- Core handler logic lives in `rust/src/tools/ctx_*.rs` — keep it pure and
deterministic (#498). See `ctx_explore.rs` for the citation-returning
exploration pattern (BM25 + static graph + AST, bounded turns, no session writes).
- The MCP adapter (implements `McpTool`) lives in `rust/src/tools/registered/ctx_*.rs`:
schema via `tool_def`, arg parsing, `ToolOutput`. Register it in
`rust/src/server/registry.rs` and bump the count SSOT in `rust/src/server/mod.rs`
(`test_registry_tool_count_ssot`).
- For a CLI surface, add `rust/src/cli/<name>_cmd.rs` and route it in
`rust/src/cli/dispatch/mod.rs` (+ `dispatch/help.rs`).
- Regenerate and commit the SSOT artifacts:
`cargo run --example gen_mcp_manifest --features dev-tools` and
`cargo run --example gen_docs --features dev-tools`.
- If you change the public tool surface, also update `LEANCTX_FEATURE_CATALOG.md`
(SSOT snapshot). The `entrypoints_wired`, `mcp_manifest_up_to_date` and
`reference_docs_drift` tests gate the wiring end to end.
### Add an addon to the registry
An addon entry in `rust/data/addon_registry.json` ships **executable trust** to
every user (a `stdio` addon runs code on their machine; an `http` addon receives
their context). Registry submissions are therefore reviewed like a security
change, not a docs change. See the
[addon manifest contract](docs/contracts/addon-manifest-v1.md#security-model).
**Your submission must:**
1. Use a unique slug `[a-z0-9-]` and fill `author`, `homepage`, `license`,
`description` (the CI validator rejects installable entries that don't).
2. **Pin the upstream.** No `latest`, no `npx/uvx`-without-a-version. The exact
command + version must be reproducible.
3. Not shell out (`sh -c`, `bash -c`), fetch-and-exec (`curl`, `wget`), or use a
non-HTTPS `url`. The validator flags all of these.
4. Point `homepage` at **public, inspectable source** for the MCP server.
5. Default to the **community** tier (`verified` stays `false`) — verification is
conferred by review, never self-asserted.
Run the validator locally — it runs in CI on every change to the registry:
```bash
cd rust && cargo test --lib addons::registry
```
**Maintainer review checklist (binding):**
- [ ] Source is public and the MCP server's behaviour matches its description.
- [ ] Command/args/url are pinned and reproducible; no shell/fetch primitives.
- [ ] `env` / `headers` carry no embedded secrets; any required secret is the
user's to supply, documented on the homepage.
- [ ] License is a real SPDX id and compatible with redistribution of the entry.
- [ ] `verified = true` requires **two** maintainer approvals **and** a clean
run with **no** `warn`/`danger` finding. Otherwise it stays community-tier.
- [ ] When in doubt, merge as a **listed** entry (no `[mcp]` block) first.
### Docs & examples
- Prefer real, runnable examples (no mock data)
- If you add a new example app, add it under `cookbook/examples/` and ensure it talks to a real `lean-ctx serve` instance
## Issues
- If your issue was closed but the problem persists, comment `/reopen` on it — as the original author, this reopens the issue automatically (GitHub itself does not let authors reopen maintainer-closed issues). The command is matched anywhere in your comment, so "Please `/reopen`" works too; issues closed as *not planned* stay a maintainer call
- Issues closed as *not planned* are maintainer decisions and are not reopened automatically; a comment is still welcome
## Pull requests
- Keep PRs focused (one theme per PR)
- Include a short test plan (commands you ran)
- If relevant, include a small “before/after” token-savings snippet
## Contributor License Agreement (CLA)
Before your first pull request can be merged, you need to sign our
[Contributor License Agreement](CLA.md). It is a one-time, automated step: the
CLA Assistant bot comments on your PR, and you sign by replying:
> I have read the CLA Document and I hereby sign the CLA
The CLA keeps lean-ctx Apache-2.0 for everyone while allowing the maintainer to
relicense (e.g. for the hosted/commercial offering). The free, open-source
runtime for individual developers stays free — that commitment is written into
the CLA itself (§8).
## License
lean-ctx is distributed under the Apache License 2.0; by contributing, your
contributions are licensed to the public under the same terms (see the [CLA](CLA.md)
for the full grant).
+61
View File
@@ -0,0 +1,61 @@
# LeanCTX — Ecosystem Overview
> Product vision: [`VISION.md`](VISION.md)
Software ate the world. Agents are eating software. And every agent is exactly
as good as the context it is given — context decides what an agent knows, what
it may do, and what it provably did. Today that context is unmanaged: untyped
markdown, copy-pasted prompts, vendor-locked memory, zero provenance.
**LeanCTX makes context infrastructure**: efficient, verifiable, portable,
organizational — managed with the same rigor as code.
## One product, three pillars
| Pillar | What it does | Who it serves |
|--------|-------------|---------------|
| **Engine** | Active token reduction — compression, MCP tools, agent hooks, local dashboard | Individual developers |
| **Gateway** | Org-wide LLM proxy — usage tracking, budget enforcement, FinOps, compliance | Companies, Ops, FinOps, CISOs |
| **Cloud** | Hosted services — accounts, sync, team provisioning, billing, registry | Teams, Pro users, managed enterprise |
All three pillars ship in **one binary** (`lean-ctx`). Every feature works
self-hosted for free. Cloud and Enterprise tiers add hosting, support, and
managed infrastructure — never features.
### Engine
The developer-facing context compression layer: 10 read modes, 95+ shell
compression patterns, tree-sitter AST for 27 languages, semantic search with
hybrid BM25 + dense retrieval, session continuity protocol (~400 tokens instead
of ~50K cold start). Ships as MCP tools (`ctx_read`, `ctx_compose`,
`ctx_search`, …) and transparent agent hooks.
### Gateway
The org-wide LLM reverse proxy: intercepts Anthropic, OpenAI, Gemini and
ChatGPT traffic, compresses prompts in-flight, meters per-request cost
attribution, enforces budgets, and provides FinOps dashboards. Self-hosted with
`--features gateway-server`, no license required.
### Cloud
The hosted coordination layer: user accounts, team provisioning, knowledge
sync, billing edge, and the context package registry. Enables Pro/Team/
Enterprise tiers without requiring self-hosted infrastructure.
## Companion projects
| Project | Role |
|---------|------|
| **ctxpkg.org** | Open standard for signed, typed context packages (`.ctxpkg`) |
| **ctxpkg.com** | Registry and marketplace for context packages |
## Doctrines
- **Zero telemetry, absolutely** — nothing leaves a machine automatically;
explicit, locally computed, user-invoked shares only.
- **Trust is never for sale** — no paid placement, ranking or verification on
any surface.
- **Local-Free Invariant** — every feature works self-hosted for free;
commercial tiers add hosting and support, never capabilities.
- **Distilled, typed, signed knowledge only** — never raw transcripts.
+55
View File
@@ -0,0 +1,55 @@
<!-- lean-ctx-owned: PROJECT-LEAN-CTX.md v1 -->
<!-- lean-ctx-rules -->
<!-- version: 8 -->
CRITICAL: ALWAYS use lean-ctx ctx_* tools instead of native equivalents. This is NOT optional.
ACTUALLY EMIT the ctx_* tool call (ctx_compose first) — describing a tool is not calling it.
MANDATORY MAPPING:
• Read/cat -> ctx_read(path, mode)
• Grep -> ctx_search(pattern, path)
• Shell/bash -> ctx_shell(command)
• Glob/find -> ctx_glob(pattern)
• ls/find -> ctx_tree(path, depth)
NEVER use native Read/Grep/Shell/Glob when a ctx_* equivalent exists. SELF-CORRECT: the moment you reach for one, stop and call the ctx_* tool instead.
Tool selection by intent:
• Orient / understand code (call FIRST) -> ctx_compose
• Read a file -> ctx_read(path, mode=signatures|map|full); edit after reading -> ctx_patch
• Exact symbol -> ctx_search(action=symbol); pattern -> ctx_search; by meaning -> ctx_search(action=semantic)
• Files by glob -> ctx_glob; structure -> ctx_tree; callers/impact -> ctx_callgraph
• Verify after edits -> ctx_shell(test/build); memory -> ctx_session / ctx_knowledge
Semantic questions -> search tools, not whole-file reads: reading more ≠ understanding more.
AGENT LOOP (phase -> tool):
• Orient — understand before acting -> ctx_compose
• Find — exact symbol by name -> ctx_search(action=symbol)
• Read — a file, structurally -> ctx_read(mode=signatures|map)
• Locate — a pattern across files -> ctx_search
• Trace — callers / callees / blast radius -> ctx_callgraph
• Verify — after an edit -> ctx_shell(test/build) + native lints
Anti-patterns — do NOT:
• Chain ctx_search -> ctx_read -> ctx_search(action=symbol) — one ctx_compose replaces all three
• Use ctx_read(mode=full) for orientation — use mode=signatures
• Use ctx_callgraph/ctx_graph for const/static/variable refs — they track call edges and file deps only; use ctx_search instead
NAVIGATION PARADOX: reading more ≠ understanding more.
• Semantic question ("where/how is X handled?") -> ctx_search (BM25) + ctx_search(action=semantic) (meaning), not whole-file reads
• Hidden architectural deps (who calls this, what breaks) -> ctx_callgraph / ctx_graph — for these only
• Navigate structure (signatures, symbols) before reading entire files
PARALLEL: fire independent tool calls in the SAME turn — ctx_compose bundles multiple lookups into one call.
Auto: preload/dedup/compress run in background. ctx_session=memory, ctx_knowledge=facts, ctx_shell raw=true=uncompressed. Full guide: LEAN-CTX.md
RECOVER: compressed output is reversible — never re-read line-by-line. Need full/exact? Read the shown file path with any tool (no MCP), or ctx_read(mode=full|raw=true); [Archived]/tee/firewall → ctx_expand(id=...).
CEP v1: 1.ACT FIRST 2.DELTA ONLY (Fn refs) 3.STRUCTURED (+/-/~) 4.ONE LINE PER ACTION 5.QUALITY ANCHOR
OUTPUT: never echo tool output, no narration comments, show only changed code.
TOOL PREFERENCE (END): ctx_compose>chain ctx_read>Read ctx_shell>Shell ctx_search>Grep ctx_glob>Glob ctx_tree>ls | Edit/Write/Delete=native
<!-- /lean-ctx-rules -->
+372
View File
@@ -0,0 +1,372 @@
# LeanCTX Feature Catalog (SSOT Snapshot)
**Version:** `3.8.1`
**Updated:** `2026-05-15`
**Primary Sources:** `website/generated/mcp-tools.json`, `rust/src/tool_defs/granular.rs`, `README.md`
---
## Purpose
This catalog is the single feature inventory for LeanCTX at release/runtime level:
- Which MCP tools exist now
- Which entry points are canonical vs deprecated aliases
- Which read modes are supported
- Which shell pattern modules exist
- Which CLI commands are available
- Which capabilities are part of the shipped product surface
---
## Runtime Surface (Current)
- Granular MCP tools: **81**
- Unified MCP tools: **5**
- MCP Resources: **5**
- MCP Prompts: **5**
- Dynamic Tool Categories: **6**
- Shell pattern modules: **56** (fd, just, ninja, clang, extended cargo run/bench added in 3.4.x)
- CLI commands: **80+** (knowledge, overview, compress, serve --daemon/--stop/--status, session, pack create/list/info/export/import/install/remove/auto-load subcommands added in 3.4.x)
- Read modes: **10** (`auto`, `full`, `map`, `signatures`, `diff`, `aggressive`, `entropy`, `task`, `reference`, `lines:N-M`)
- Positioning: Context Runtime for AI agents (shell hook + context server + setup integrations)
---
## Unified MCP Tools (5)
- `ctx`
- `ctx_read`
- `ctx_search`
- `ctx_shell`
- `ctx_tree`
---
## Granular MCP Tools (81)
### A) Read / Search / IO Surface
- `ctx_read`
- `ctx_multi_read`
- `ctx_smart_read`
- `ctx_tree`
- `ctx_search`
- `ctx_semantic_search`
- `ctx_shell`
- `shell` _(alias of `ctx_shell` — gives Codex Desktop/Cloud the same shell-output compression; registered for all MCP clients)_
- `ctx_edit`
- `ctx_patch` _(hash-anchored line edits — `LINE:HASH` anchors from `ctx_read mode=anchored`, no exact-recall)_
- `ctx_delta`
- `ctx_dedup`
- `ctx_fill`
- `ctx_outline`
- `ctx_symbol`
- `ctx_routes`
- `ctx_context`
- `ctx_compose`
- `ctx_explore` _(FastContext-style bounded multi-turn exploration → `path:start-end` citations; locates code across files at a fraction of `ctx_compose`'s tokens)_
### B) Architecture / Analysis / Discovery
- `ctx_graph` _(actions: build, related, symbol, impact, status, enrich, context, diagram)_
- `ctx_callgraph` _(direction=callers|callees)_
- `ctx_refactor` _(LSP: rename, references, definition, implementations)_
- `ctx_architecture`
- `ctx_impact`
- `ctx_quality` _(navigability score + cognitive-complexity hotspots + estimated token "quality tax"; agent twin of `lean-ctx health`)_
- `ctx_review`
- `ctx_pack`
- `ctx_index`
- `ctx_artifacts`
- `ctx_intent`
- `ctx_task`
- `ctx_overview`
- `ctx_preload`
- `ctx_prefetch`
- `ctx_discover`
- `ctx_analyze`
### C) Session / Knowledge / Multi-Agent
- `ctx_session`
- `ctx_knowledge`
- `ctx_agent`
- `ctx_share`
- `ctx_handoff`
- `ctx_workflow`
- `ctx_feedback`
#### Knowledge CLI (`lean-ctx knowledge`)
Full CLI parity with MCP `ctx_knowledge`:
- `remember <value> --category <c> --key <k>` — store a fact
- `recall [query] [--category <c>] [--mode auto|semantic|hybrid]` — retrieve facts
- `search <query>` — cross-project knowledge search
- `export [--format json|jsonl|simple] [--output <path>]` — export knowledge (stdout or file)
- `import <path> [--merge replace|append|skip-existing] [--dry-run]` — import from JSON/JSONL
- `remove --category <c> --key <k>` — remove a fact
- `consolidate [--all]` — import latest session if present, run lifecycle, then leave 25% facts/history/procedures capacity free; `--all` repeats this for every stored project root
- `status` — knowledge base summary
- `health` — health report with quality metrics
- `lifecycle` — read-only lifecycle/capacity report
Import supports three formats: native `ProjectKnowledge` JSON, simple `[{category, key, value}]` array, and JSONL (one fact per line). The `simple` format serves as the community interop schema for migration from other tools.
### D) Compression / Metrics / Runtime Ops
- `ctx_cache`
- `ctx_compress`
- `ctx_expand` _(actions: retrieve, list, search_all — FTS5 cross-archive fulltext search)_
- `ctx_call`
- `ctx_compress_memory`
- `ctx_metrics`
- `ctx_cost`
- `ctx_heatmap`
- `ctx_gain` _(actions: wrapped, summary, delta)_
- `ctx_execute`
- `ctx_benchmark`
- `ctx_compare` _(preview compression — original vs the bytes lean-ctx would emit + token counts and line diff, read-only)_
- `ctx_response`
- `ctx_tools` _(MCP Tool-Catalog Gateway — actions: find, call, list, refresh; routes/proxies unlimited downstream MCP servers at constant context cost)_
---
## MCP Protocol Capabilities
### MCP Resources (5)
Subscribe-capable resources, gated by client capabilities:
- `lean-ctx://context/summary`
- `lean-ctx://context/pressure`
- `lean-ctx://context/plan`
- `lean-ctx://context/pinned`
- `lean-ctx://context/bounce`
### MCP Prompts (5)
Appear as slash commands in supporting IDEs:
- `/context-focus`
- `/context-review`
- `/context-reset`
- `/context-pin`
- `/context-budget`
### Elicitation
Rate-limited context decisions triggered by:
- Pressure >90%
- Large files >5k tok
- Budget exhaustion
Fallback hints emitted for non-supporting IDEs.
### Dynamic Tool Categories (6)
On-demand loading via `ctx_load_tools` + `notifications/tools/list_changed`:
- **core** (~27 tools, always loaded)
- **arch**
- **debug**
- **memory**
- **metrics**
- **session**
`ctx_load_tools` _(actions: load, unload, list)_ — explicit category management at runtime. After each change, `notifications/tools/list_changed` is sent to subscribed clients.
---
## Intelligence Layer
### Context Gate (Active)
Pre-dispatch mode override:
- **Overlay override** (pin/exclude/set_view)
- **Pressure-based auto-downgrade** (ForceCompression: full→map, EvictLeastRelevant: map→signatures)
- Bounce-prevention
- Intent-target (with real task from SessionState)
- Graph-proximity
- Knowledge-relevance
Post-dispatch:
- Ledger recording (with real task Φ computation)
- Reinjection plan (downgrade existing "full" entries to "map" under pressure)
- Eviction/elicitation hints
- `notifications/resources/updated` on significant ledger changes
### Bounce Detection
Tracks wasted tokens from compressed→full re-reads:
- Per-extension bounce rates
- Adjusts savings metrics to report honest numbers
### Client Capability Detection
Runtime detection of 9 IDE clients:
- Cursor, Claude Code, CodeBuddy, Windsurf, Zed, VS Code Copilot, Kiro, Codex, Antigravity, Gemini CLI
Tier 14 classification determines feature gating for resources, prompts, elicitation, and dynamic tools.
---
## Removed Aliases (v3.6.1+)
Previously deprecated aliases have been removed. Use the canonical tools:
- `ctx_callgraph direction=callers` (was: ctx_callers)
- `ctx_callgraph direction=callees` (was: ctx_callees)
- `ctx_graph action=diagram` (was: ctx_graph_diagram)
- `ctx_gain action=wrapped` (was: ctx_wrapped)
---
## Capabilities (3.4.x Additions)
### Daemon Mode
- Unix Domain Socket server via `lean-ctx serve --daemon`
- Control via `--stop`, `--status` flags
- Persistent background process for lower-latency MCP serving
### Multi-Tokenizer Support
- `o200k_base` (GPT-4o, default)
- `cl100k_base` (GPT-4 / GPT-3.5)
- Gemini tokenizer
- Llama tokenizer
### Hook Modes
- `HookMode::Mcp` — MCP server only (IDE-extension agents without reliable shell hooks)
- `HookMode::Hybrid` — MCP server + shell hooks (default where shell access exists)
### Smart Mode Selection in Setup
- `lean-ctx setup` auto-detects editor and agent, selects optimal hook mode
### SKILL.md Auto-Installation
- `lean-ctx init` writes `SKILL.md` to agent-specific skill directories
- Auto-detects Cursor, Claude Code, CodeBuddy, Codex, Gemini CLI, Kiro skill paths
### Compressed Output Cache
- `map` and `signatures` read modes cache compressed output
- Re-reads of compressed representations cost ~13 tokens
### Intent-Aware Read Mode Selection
- `ctx_read mode=auto` uses task intent to select optimal compression
- Factors: file type, file size, task signal, access history
### Prefix-Cache-Friendly Output Ordering
- Imports and type definitions emitted before function bodies
- Stable ordering maximizes LLM provider KV-cache hits
### Field-Wise Profile Merge
- Context profiles merged field-by-field (not replaced wholesale)
- Allows layered profile composition (base + project + role)
### Token Counting Deduplication
- Cross-file shared block detection via `ctx_dedup`
- Deduplicated blocks counted once in budget calculations
### Graph-Powered Context OS (3.4.7)
#### Multi-Edge Graph Queries
- Property Graph queries traverse `imports`, `calls`, `exports`, `type_ref`, `tested_by`
- Weighted BFS: imports=1.0, calls=0.8, exports=0.7, type_ref=0.5, tested_by=0.4
- New query: `related_files(path, limit)` — scored file neighbors
- New query: `file_connectivity(path)` — edge-type breakdown per file
#### Graph-Aware File Reads
- Every `ctx_read` includes a `[related: ...]` hint from the Property Graph
- Scored related files shown with relationship strength (e.g., `config.rs (80%)`)
- Agent understands file context immediately without extra tool calls
#### Calls + Exports Edge Materialization
- Graph build writes `EdgeKind::Calls` edges (symbol-to-symbol and file-to-file)
- Graph build writes `EdgeKind::Exports` for exported symbols (symbol-to-file)
- Impact analysis now tracks function-level blast radius, not just file-level
#### Incremental Graph Update
- `ctx_impact action="update"` uses `git diff --name-only` to detect changed files
- Only changed files are re-indexed (remove_file_nodes + re-parse)
- Deleted files have their nodes removed automatically
- Full build hints when incremental update is available
#### Session Survival Engine
- `build_compaction_snapshot()` generates structured recovery XML:
- `<recovery_queries>`: executable `ctx_read`/`ctx_search` commands
- `<knowledge_context>`: `ctx_knowledge recall` queries from task keywords
- `<graph_context>`: dependency cluster references for touched files
- Budget-aware with automatic section shrinking
#### Knowledge-Enriched Overview
- `ctx_overview` includes top relevant Knowledge facts for the current task
- Graph architectural hotspots: files ranked by edge count (imports + calls)
#### Hybrid Search Fusion (RRF)
- `ctx_semantic_search` combines 3 signals via Reciprocal Rank Fusion:
- BM25 (lexical), Dense Embeddings (semantic), Graph Proximity (structural)
- `score = sum(1 / (60 + rank_i))` over all signals
- Graph neighbors of recently touched files get score boost
#### Progressive Search Throttling
- Per-session tracker for repeated pattern+path combinations
- Calls 1-3: normal, 4-6: hint to use `ctx_knowledge remember`, 7+: throttle hint
- Auto-reset after 5 minutes idle
#### Sandbox-First Routing
- `ctx_shell` output >5KB: hint to use `ctx_execute` for compression
- `ctx_read` full mode >10K tokens: hint to use `map`/`aggressive`
- Once-per-session deduplication
#### Terse Mode
- `ctx_session action="configure" terse=true` enables concise response mode
- Injects instruction into resume block: focus on code/actions, avoid filler
- Survives context compaction via snapshot
#### Context Package System (3.4.7)
Context packages bundle Knowledge, Graph, Session, Patterns, and Gotchas into portable `.ctxpkg` files that can be shared, versioned, and auto-loaded across projects.
##### Package Layers
- **Knowledge**: facts, patterns, consolidated insights from ProjectKnowledge
- **Graph**: nodes + edges from the Property Graph (full SQLite export)
- **Session**: task description, findings, decisions, next steps, files touched
- **Patterns**: project patterns extracted from knowledge
- **Gotchas**: gotcha entries with triggers, resolutions, file patterns
##### CLI Commands (9 subcommands under `lean-ctx pack`)
- `create` — build package from current project context (knowledge, graph, session, gotchas)
- `list` — list all installed packages with layers, size, auto-load status
- `info` — detailed package view (stats, integrity, provenance, estimated tokens)
- `remove` — remove package from local registry
- `export` — export to portable `.ctxpkg` file (JSON bundle with SHA-256 integrity)
- `import` — import from `.ctxpkg` file into local registry
- `install` — apply package to current project (merge knowledge, import graph, import gotchas)
- `auto-load` — enable/disable automatic loading on `ctx_overview` session start
- `pr` — existing PR context pack (unchanged)
##### Premium Features
- **SHA-256 Content Integrity**: canonical compact JSON hashing, verified on every load
- **Atomic Writes**: tmp + rename pattern prevents corruption
- **Knowledge Merge**: duplicate detection (category + key + value), confidence capped at 0.8 for imports
- **Graph Overlay**: nodes and edges imported directly into the SQLite Property Graph
- **Gotcha Import**: ID-based dedup, severity mapping, confidence capping
- **Auto-Load**: packages flagged `auto_load=true` loaded on every `ctx_overview` call
- **Schema Versioning**: `CONTEXT_PACKAGE_V1_SCHEMA_VERSION` in `contracts.rs`
- **Compression Stats**: gzip-based compression ratio tracked in manifest
---
## Notes For Releases
- Tool counts and tool names must match `website/generated/mcp-tools.json`.
- Shell pattern module count must match `rust/src/core/patterns/mod.rs` module list.
- Any new tool or alias change requires synchronized updates in:
- `README.md` and relevant package READMEs
- `rust/src/templates/*` where applicable
- this catalog
- Historical counts in old CHANGELOG entries remain unchanged by design.
+191
View File
@@ -0,0 +1,191 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to the Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by the Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding any notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
Copyright 2026 Yves Gugger
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+40
View File
@@ -0,0 +1,40 @@
.PHONY: setup-hooks install dev test preflight preflight-fast help
# ── Setup ─────────────────────────────────────────────────
setup-hooks: ## Configure git to use .githooks/ for hooks
git config core.hooksPath .githooks
@echo "Git hooks configured: .githooks/"
# ── Build & Install ──────────────────────────────────────
install: ## Build release + install to ~/.local/bin
cd rust && cargo install --path . --force --locked --root "$$HOME/.local"
@echo "Installed: $$(lean-ctx --version)"
dev: ## Quick debug build + copy to ~/.local/bin
cd rust && cargo build
@mkdir -p "$$HOME/.local/bin"
cp rust/target/debug/lean-ctx "$$HOME/.local/bin/lean-ctx"
@echo "Dev installed: $$(lean-ctx --version)"
test: ## Run all Rust tests + clippy
cd rust && cargo test && cargo clippy
# ── CI-parity gate ───────────────────────────────────────
# Mirrors .github/workflows/ci.yml so green-here => green-in-CI for the
# deterministic jobs (fmt/clippy/doc/gen_docs/cross-platform compile).
preflight: ## Full local CI-parity gate (fmt, clippy, doc, gen_docs, win-check, lib tests)
scripts/preflight.sh full
preflight-fast: ## Static CI-parity gate (no full test run) — what pre-push runs
scripts/preflight.sh fast
# ── Help ──────────────────────────────────────────────────
help: ## Show this help
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}'
.DEFAULT_GOAL := help
+16
View File
@@ -0,0 +1,16 @@
lean-ctx
Copyright 2026 Yves Gugger
Licensed under the Apache License, Version 2.0.
This product includes software developed by:
- Yves Gugger (https://github.com/yvgude)
Third-party contributions:
- frpboy: MCP performance fix, responsive dashboard
- Victor Bressani de Mello (Chokitus): Kotlin graph support
- Dustin Reynolds: various improvements
- Yassine El Ouni (sinouw): various improvements
- Julien Bodin (jbodin): various improvements
- Tarek Hamlaoui: various improvements
- Almario Miano: various improvements
+694
View File
@@ -0,0 +1,694 @@
<div align="center">
<pre>
██╗ ███████╗ █████╗ ███╗ ██╗ ██████╗████████╗██╗ ██╗
██║ ██╔════╝██╔══██╗████╗ ██║ ██╔════╝╚══██╔══╝╚██╗██╔╝
██║ █████╗ ███████║██╔██╗ ██║ ██║ ██║ ╚███╔╝
██║ ██╔══╝ ██╔══██║██║╚██╗██║ ██║ ██║ ██╔██╗
███████╗███████╗██║ ██║██║ ╚████║ ╚██████╗ ██║ ██╔╝ ██╗
╚══════╝╚══════╝╚═╝ ╚═╝╚═╝ ╚═══╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝
</pre>
### **Control what your AI can see.**
**LeanCTX — Lean Context Engineering for AI agents**
LeanCTX — short for **Lean Context** — is the context engineering layer for
AI agents. It runs as a single local binary between your agents and everything
they touch — your code, shell, data, and the model itself: it **decides** what
they read, **compresses** what they send (an optional local proxy shrinks every
request — system prompt, history and tool results — prompt-cache-safe),
**remembers** what they learn, **guards** what they touch — and **proves** what
they save with a signed, verifiable savings ledger. The result: 6090% fewer
tokens.
Zero config required.
Local-first.
| Problem | With LeanCTX |
|---------|-------------|
| Repeated file reads: ~2000 tokens each | Cached re-reads: **~13 tokens** |
| Raw `git status`: ~800 tokens | Compressed: **~120 tokens** |
| Every turn re-sends the whole history | Proxy compresses each request, **prompt-cache-safe** |
| Context resets every chat | Session memory persists across chats |
| No visibility into context usage | Real-time dashboard + budget control |
---
<p>
<a href="https://github.com/yvgude/lean-ctx/stargazers"><img src="https://img.shields.io/github/stars/yvgude/lean-ctx?style=social" alt="GitHub Stars"></a>&nbsp;&nbsp;
<a href="https://github.com/yvgude/lean-ctx/actions/workflows/ci.yml"><img src="https://github.com/yvgude/lean-ctx/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
<a href="https://github.com/yvgude/lean-ctx/actions/workflows/security-check.yml"><img src="https://github.com/yvgude/lean-ctx/actions/workflows/security-check.yml/badge.svg" alt="Security"></a>
<a href="https://crates.io/crates/lean-ctx"><img src="https://img.shields.io/crates/v/lean-ctx?color=%23e6522c" alt="crates.io"></a>
<a href="https://crates.io/crates/lean-ctx"><img src="https://img.shields.io/crates/d/lean-ctx?color=%23e6522c" alt="Downloads"></a>
<a href="https://www.npmjs.com/package/lean-ctx-bin"><img src="https://img.shields.io/npm/v/lean-ctx-bin?label=npm&color=%23cb3837" alt="npm"></a>
<a href="https://aur.archlinux.org/packages/lean-ctx"><img src="https://img.shields.io/aur/version/lean-ctx?color=%231793d1" alt="AUR"></a>
<a href="https://pi.dev/packages/pi-lean-ctx"><img src="https://img.shields.io/badge/Pi.dev-pi--lean--ctx-6366f1?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJ3aGl0ZSI+PHRleHQgeD0iNCIgeT0iMTgiIGZvbnQtc2l6ZT0iMTYiIGZvbnQtZmFtaWx5PSJzZXJpZiI+z4A8L3RleHQ+PC9zdmc+" alt="Pi.dev"></a>
<a href="LICENSE"><img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" alt="License"></a>
<a href="https://discord.gg/pTHkG9Hew9"><img src="https://img.shields.io/badge/Discord-Join-5865F2?logo=discord&logoColor=white" alt="Discord"></a>
<a href="https://x.com/leanctx"><img src="https://img.shields.io/badge/𝕏-Follow-000000?logo=x&logoColor=white" alt="X/Twitter"></a>
<img src="https://img.shields.io/badge/Telemetry-Opt--in%20Only-brightgreen?logo=shield&logoColor=white" alt="Opt-in Telemetry">
</p>
<p>
<a href="https://leanctx.com">Website</a>&nbsp;&nbsp;·&nbsp;&nbsp;<a href="https://leanctx.com/docs/getting-started">Docs</a>&nbsp;&nbsp;·&nbsp;&nbsp;<a href="#get-started-60-seconds">Install</a>&nbsp;&nbsp;·&nbsp;&nbsp;<a href="#use-it-from-your-own-code-sdks">SDKs</a>&nbsp;&nbsp;·&nbsp;&nbsp;<a href="#real-world-scenarios">Scenarios</a>&nbsp;&nbsp;·&nbsp;&nbsp;<a href="#demo">Demo</a>&nbsp;&nbsp;·&nbsp;&nbsp;<a href="#benchmarks">Benchmarks</a>&nbsp;&nbsp;·&nbsp;&nbsp;<a href="cookbook/README.md">Cookbook</a>&nbsp;&nbsp;·&nbsp;&nbsp;<a href="SECURITY.md">Security</a>&nbsp;&nbsp;·&nbsp;&nbsp;<a href="CHANGELOG.md">Changelog</a>
</p>
</div>
---
> **Control what your AI can see.** LeanCTX — short for **Lean Context** — is the **context engineering layer** for AI agents: one local Rust binary that decides what your agents read, compresses what they send to the model, remembers what they learn, guards what they touch — and proves what they save.
> Token savings are the receipt. Intelligence is the product. Works with **Cursor, Claude Code, Copilot, Windsurf, Codex, Gemini** and 30+ other agents — no config needed.
<p align="center"><strong>See it in action:</strong></p>
<table>
<tr>
<td align="center" width="33%">
<img src="assets/leanctx-demo.gif" width="320" alt="Map-mode file read + compressed git output demo">
<br/>
<strong>Read + Shell</strong>
<br/>
Map-mode reads + compressed CLI output
</td>
<td align="center" width="33%">
<img src="assets/leanctx-gain.gif" width="320" alt="lean-ctx gain live dashboard demo">
<br/>
<strong>Gain (live)</strong>
<br/>
Tokens + USD savings in real time
</td>
<td align="center" width="33%">
<img src="assets/leanctx-benchmark.gif" width="320" alt="lean-ctx benchmark report demo">
<br/>
<strong>Benchmark proof</strong>
<br/>
Measure compression by language + mode
</td>
</tr>
</table>
<p align="center"><sub>All GIFs are generated from reproducible VHS tapes in <code>demo/</code>.</sub></p>
## Why developers use LeanCTX
- **Longer useful coding sessions** — less context waste = more room for actual code reasoning
- **Lower API costs** — 6090% fewer tokens on reads and shell output, cached re-reads cost ~13 tokens
- **No more "I already showed you this file"** — session memory persists across chats
- **Works with your existing setup** — one `lean-ctx setup` command, no config changes needed
- **Full visibility** — see exactly where your context window budget goes
- **Model-agnostic & yours** — swap OpenAI/Anthropic/Gemini freely; your context and memory stay local and portable, never locked in a vendor's black box
---
<p align="center">
<strong>Saves you tokens?</strong> <a href="https://github.com/yvgude/lean-ctx">Give it a star</a> — it helps others discover LeanCTX.
</p>
---
## Why now — own your context
Models are converging on commodity. The durable edge isn't *which* model you call — it's your **context**: what your agents read, what they remember, and what you can prove. And the layer that optimizes and *owns* that context can't come from the vendor that bills per token or keeps your memory in a black box — it has to sit on your side.
That's the shift behind "agent entities" that live in your chat and remember your company (Claude in Slack, ClickUp Brain): a **context login, not a model login** — you end up renting your own company knowledge back. LeanCTX is the opposite layer. It keeps the moat yours: local-first, portable (`.ctxpkg`), and model-agnostic — swap OpenAI, Anthropic or Gemini without losing context or cache. **Own your context; don't rent it back.**
---
## What it does — the four dimensions of context
LeanCTX treats context as a managed resource, not an afterthought. One binary
covers the four dimensions that decide how well an AI agent actually performs:
### 1. Compression — input efficiency
Your AI agent reads files and runs commands. LeanCTX compresses both automatically.
- **File reads**: 10 read modes (`full`, `map`, `signatures`, `diff`, `lines:N-M`, `density:X`, …) — cached re-reads cost ~13 tokens
- **Target density** (`density:0.4`): SDE-style budget compression — keeps the highest-entropy lines until ~40% of the original tokens remain, deterministic
- **JIT disclosure**: `signatures` carries line spans and points at `lines:N-M` for targeted expansion — outline first, bodies on demand
- **Shell output**: 95+ shell-output patterns compress git, npm, cargo, docker, kubectl, terraform and more (270 passthrough rules)
- **Tree-sitter AST**: structural understanding for 27 languages — not just text compression
- **Reversible by design (CCR)**: compression never *discards* content — pruned or truncated payloads move to a content-addressed store with a deterministic handle, so the model can pull the original bytes back on demand via `ctx_expand`, `ctx_retrieve`, an in-band marker, or `GET /v1/references/{id}`. [Five recovery paths →](docs/comparisons/vs-headroom.md#reversibility)
### 2. Routing — the right fidelity per read
Not every file needs the same depth. LeanCTX sends the signal, not the noise.
- **10 read modes**: from full content down to AST signatures and entropy-filtered views
- **Adaptive `ModePredictor`**: learns the optimal read mode per file type from past sessions
- **`IntentEngine`**: classifies query complexity so simple lookups stay cheap
### 3. Memory — context that persists
Context doesn't disappear between chats anymore.
- **Session memory (CCP)**: persist task/facts/decisions across chats — structured recovery queries survive compaction
- **Knowledge graph**: temporal facts with validity windows, episodic + procedural memory
- **Property Graph**: multi-edge code graph (imports, calls, exports, type_ref) powers impact analysis and search ranking
- **Yours, not the vendor's**: memory stays local and portable — export it as a `.ctxpkg` package and move it across machines or models, instead of locking it in a vendor's black box
### 4. Verification — control what reaches the model
Performance is accuracy, not just speed. You stay in control of the window.
- **Context Manager**: browser dashboard with real-time token tracking, compression stats, utilization gauge
- **Budgets & SLOs**: profiles, roles, per-agent budgets, and throttling policies
- **Context Proof** (`ctx_proof`, `ctx_verify`): 4-layer verification engine with CI drift gates
<details>
<summary><strong>Full feature list (81 MCP tools)</strong></summary>
- **Web & Research** (`ctx_url_read`): pull a public web page, PDF, or YouTube transcript into context as compressed, citation-backed text — `facts`/`quotes` return claims with a confidence score + source URL, relevance-ranked research-compression distils to a token budget, SSRF-guarded (http/https only)
- **Graph-Powered Intelligence**: hybrid search (BM25 + embeddings + graph proximity via RRF), incremental git-diff updates
- **LSP Refactoring** (`ctx_refactor`): language-server-powered rename, references, go-to-definition via rust-analyzer, typescript-language-server, pylsp, gopls
- **Multi-Agent** (`ctx_agent`, `ctx_handoff`): agent handoff with context transfer bundles, diary system, synchronized shared state
- **Archive Full-Text Search** (`ctx_expand search_all`): FTS5-powered cross-archive search over all previously archived tool outputs
- **PR Context Packs**: `lean-ctx pack --pr` builds a PR-ready context pack (changed files, related tests, impact, artifacts)
- **Context Packages**: `lean-ctx pack create` bundles Knowledge + Graph + Session into portable `.ctxpkg` files with SHA-256 integrity
- **Context Time Machine**: `lean-ctx snapshot create|list|show|verify|restore|publish|import` — git-anchored, ed25519-signed snapshots of the layer state (lineage, ledger Φ, ROI, session) on an append-only timeline; replay them in the dashboard, `restore` to resume a session (and `--git` to check out the commit), or `publish`/`import` a signed snapshot to share it ([concept →](docs/concepts/context-time-machine.md))
- **Observability**: `lean-ctx gain --live` for real-time savings, `lean-ctx wrapped` for weekly/monthly summaries (`gain --svg`/`--share` for a shareable card or self-hostable page), `lean-ctx watch` for TUI monitoring
- **Verified savings**: `lean-ctx savings` is an auditable, per-event ledger (tokenizer transparency, bounce-netting, tamper-evident SHA-256 chain) — local-only, on by default
- **HTTP mode**: `lean-ctx serve` for Streamable HTTP MCP + `/v1/tools/call` (used by the Cookbook + SDK)
</details>
## Addons — run the ecosystem through one gateway
You don't have to choose between LeanCTX and the other context tools you already
like. An **addon** wraps any MCP server in a tiny `lean-ctx-addon.toml` manifest;
LeanCTX runs it behind its gateway with one command, then treats what it returns
like your own reads instead of just proxying it.
```bash
lean-ctx addon search memory # browse the registry by category
lean-ctx addon add headroom # installs the upstream package + wires the MCP server, on add
lean-ctx addon list # what's wired into your gateway
```
- **One command to add** — `addon add <name>` installs the upstream package via its native manager (uv, pip, cargo, npm, brew, dotnet) and wires the MCP server into your gateway. No fork, no recompile.
- **Folded in, not just proxied** — opt-in post-processing runs addon output through the same pipeline as your code: compress to a budget, spill oversized blobs to a `ctx_expand` handle, index into BM25 / graph / knowledge. Typed adapters route specific tools straight into `ctx_expand`, `ctx_callgraph` and `ctx_knowledge`.
- **Untrusted by default** — every addon's output is scrubbed for secrets and tagged untrusted before it reaches the model. Always on, not a flag.
- **You stay in control** — pin a machine or fleet to verified-only, an allowlist, or off entirely via one `[addons]` policy a single repo can't override.
The registry spans compression (Headroom, Sophon), code intelligence (Repomix,
Serena), memory (Mem0, Cognee, Letta) and reasoning (Sequential Thinking). See the
**[addon guide](docs/guides/addons.md)** or [browse them all](https://leanctx.com/addons/).
## Where it's going
LeanCTX is growing from a single context *layer* into a full **cognitive context
layer** for whole teams: version-controlled context strategy, one unified graph, and a
governance layer across many agents.
- **Context Time Machine → hosted history** — the snapshot engine, dashboard replay, restore, and signed file-based share/import have shipped (see above); next is a `ctxpkg.com` registry for hosted, versioned context history and a side-by-side model-view git-diff replay. The temporal axis through everything LeanCTX does — it *decides, remembers, guards, proves, and replays*. ([concept →](docs/concepts/context-time-machine.md))
- **Context as Code** — declarative pipelines, profiles, and policies in TOML, versioned like infrastructure
- **Unified Context Graph** — code, tests, commits, CI runs, and knowledge entries in a single semantic graph
- **Agent Harness** — roles, budgets, and tool permissions for multi-agent governance
- **Context Observability** — SLOs on context consumption, anomaly detection, OpenTelemetry / Prometheus export
The full roadmap lives in **[VISION.md](VISION.md)**.
## How it works (30 seconds)
LeanCTX works on **two planes** — what your agents *read* and what they *send to the model*:
```
read path: AI tool → (MCP tools + shell) → lean-ctx → your repo + CLI
wire path: AI tool → lean-ctx proxy → model provider (every request, compressed)
```
- **MCP server** *(read path)*: exposes `ctx_*` tools (read modes, caching, deltas, search, memory, multi-agent)
- **Shell hook** *(read path)*: transparently compresses common commands so the LLM sees less noise
- **Request proxy** *(wire path, opt-in)*: `lean-ctx proxy enable` puts a local proxy between your agent and the model that compresses **every request** — system prompt, full history and tool results — prompt-cache-safe, with measured USD spend. It can also pin **one reasoning-effort level across OpenAI, Anthropic & Gemini** (`proxy.effort`) without breaking that cache, cut **output** tokens with a cache-safe verbosity steer plus a measured holdout, and **relocate volatile fields** (dates, UUIDs, commit SHAs) out of the cacheable prefix so a stable system prompt finally caches. Every rewrite is reversible (content-addressed recovery) and byte-stable by contract. Same layer as a standalone request-compression proxy (e.g. Headroom) — you don't need one on top.
- **Property Graph**: multi-edge code graph powers impact analysis, related file discovery, and search ranking
- **Session memory**: persists state with structured recovery so long-running work never "cold starts"
- **Context Manager**: browser dashboard for real-time visibility into what's in your context window
## Get started (30 seconds)
```bash
# 1) Install (pick one)
curl -fsSL https://leanctx.com/install.sh | sh # universal (no Rust needed)
brew tap yvgude/lean-ctx && brew install lean-ctx # macOS / Linux
npm install -g lean-ctx-bin # Node.js
cargo install lean-ctx # Rust
pi install npm:pi-lean-ctx # Pi Coding Agent
# 2) One-command setup for your agent
lean-ctx wrap cursor # or: wrap claude / wrap codex / wrap vscode
# Done. Savings appear after your AI's first lean-ctx call.
lean-ctx gain
```
`lean-ctx wrap` installs shell hooks, registers the MCP server, sets up agent hooks, starts the daemon, and verifies the connection — all in one command. Undo anytime with `lean-ctx unwrap cursor`.
<details>
<summary><strong>Alternative: full control</strong></summary>
```bash
lean-ctx onboard # connect all detected AI tools (zero prompts)
lean-ctx setup # interactive wizard with every option
```
</details>
**Building from source on Windows?** Clone the repo and run `./install.ps1` in PowerShell — it builds the release binary and installs it into Cargo's bin directory (pass `-BuildOnly` to build without installing).
<details>
<summary><strong>Troubleshooting / Safety</strong></summary>
- Disable immediately (current shell): `lean-ctx-off`
- Run a single command uncompressed: `lean-ctx -c --raw "git status"`
- Only activate in AI agent sessions: set `shell_activation = "agents-only"` in `~/.config/lean-ctx/config.toml`
- Per-project config override: create `.lean-ctx.toml` in your project root (auto-merged with global config)
- Docker projects sharing `/workspace`: create `.lean-ctx-id` with a unique name to prevent context collisions
- Update: `lean-ctx update`
- Diagnose (shareable): `lean-ctx doctor --json`
</details>
## Use it from your own code (SDKs)
Beyond the CLI, lean-ctx ships published libraries so you can call it directly from your app.
**Drop-in prompt compression — [`lean-ctx-sdk`](https://pypi.org/project/lean-ctx-sdk/) ([npm](https://www.npmjs.com/package/lean-ctx-sdk)).** Compress a chat-style `messages` array before it reaches any model — deterministic and prompt-cache friendly; images, tool-calls and ids pass through untouched.
```python
# pip install lean-ctx-sdk
from lean_ctx import compress
messages = compress(messages, model="claude-sonnet-4")
```
```ts
// npm install lean-ctx-sdk
import { compress } from "lean-ctx-sdk";
messages = await compress(messages, { model: "gpt-4o" });
```
Framework adapters included (LiteLLM, LangChain, Vercel AI SDK). → **[compress() cookbook](docs/guides/compress-sdk.md)**
**Thin `/v1` contract clients — [`lean-ctx-client`](https://pypi.org/project/lean-ctx-client/) ([npm](https://www.npmjs.com/package/lean-ctx-client) · [crates.io](https://crates.io/crates/lean-ctx-client)).** Wrap the full `/v1` tool, event and session API over the process boundary — never links the engine, so it stays stable as lean-ctx evolves.
```bash
pip install lean-ctx-client # Python (imports as `leanctx`)
npm install lean-ctx-client # TypeScript / Node
cargo add lean-ctx-client # Rust
```
Start the server with `lean-ctx serve`, then point a client at it. → **[API reference](https://leanctx.com/docs/api-reference/)**
## Real-world scenarios
LeanCTX grows with you. Below are the journeys most people actually take — each
links to a complete, function-by-function walkthrough in the
**[Reference](docs/reference/README.md)** (every CLI command and all 79 MCP
tools are documented there).
<table>
<tr>
<td width="50%" valign="top">
### 🟢 Your first 30 seconds
*"I just installed it — now what?"*
```bash
lean-ctx wrap cursor # one-command setup for your agent
lean-ctx doctor # confirm you're wired up
```
One command installs hooks, MCP registration, and verifies the connection.
**[Journey 1 — Setup & Onboarding](docs/reference/01-setup-and-onboarding.md)**
</td>
<td width="50%" valign="top">
### 📖 Coding every day
*"Stop re-reading the same files."*
```bash
lean-ctx read src/server.rs -m map # API surface, ~13 tok on re-read
lean-ctx -c "git status" # compressed shell output
```
Your agent reads less and searches smarter — automatically.
**[Journey 2 — Daily Use](docs/reference/02-daily-use.md)**
</td>
</tr>
<tr>
<td width="50%" valign="top">
### 🧠 Resume where you left off
*"My new chat forgot everything."*
```bash
lean-ctx overview # task-aware project recap
lean-ctx knowledge recall "auth" # facts that survive resets
lean-ctx knowledge consolidate # import session + compact lifecycle
lean-ctx knowledge consolidate --all # compact every project store
```
Session memory + a project knowledge graph persist across chats.
**[Journey 3 — Memory & Knowledge](docs/reference/03-memory-and-knowledge.md)**
</td>
<td width="50%" valign="top">
### 🗺️ Understand a new codebase
*"Where does this function ripple to?"*
```bash
lean-ctx graph impact src/auth.rs # blast radius
lean-ctx smells scan # code-smell hotspots
```
A multi-edge property graph powers impact analysis + ranked search.
**[Journey 4 — Code Intelligence](docs/reference/04-code-intelligence.md)**
</td>
</tr>
<tr>
<td width="50%" valign="top">
### 🔌 Providers & multi-repo
*"Pull in GitHub issues and our Postgres schema."*
```bash
lean-ctx provider list
lean-ctx serve --root ./api --root ./web # multi-repo
```
External data flows through the same consolidation pipeline.
**[Journey 5 — Advanced & Integrations](docs/reference/05-advanced.md)**
</td>
<td width="50%" valign="top">
### 🛠️ Keep it healthy
*"Update, fix, or cleanly remove."*
```bash
lean-ctx doctor --fix
lean-ctx update
```
Self-healing diagnostics; surgical uninstall that only removes its own blocks.
**[Journey 6 — Lifecycle & Troubleshooting](docs/reference/06-lifecycle.md)**
</td>
</tr>
<tr>
<td width="50%" valign="top">
### 🎛️ Take control of the window
*"Budget my context like a pro."*
```bash
lean-ctx plan "refactor billing" --budget 8000
lean-ctx compile --mode balanced
```
Phi-scored planning + knapsack compilation + a context ledger.
**[Journey 7 — Context Engineering](docs/reference/07-context-engineering.md)**
</td>
<td width="50%" valign="top">
### 🤝 Run a team of agents
*"Planner + coder + reviewer on one repo."*
```text
ctx_agent action=register role=dev
ctx_handoff action=create # baton-pass with full context
```
Shared message bus, diaries, knowledge, and deterministic handoffs.
**[Journey 8 — Multi-Agent Collaboration](docs/reference/08-multi-agent.md)**
</td>
</tr>
<tr>
<td width="50%" valign="top">
### 🏢 Share across a team / CI
*"One shared index, headless in pipelines."*
```bash
lean-ctx team serve --config team.toml
lean-ctx bootstrap # zero-prompt CI setup
```
Scoped tokens, optional cloud sync, verifiable context gates.
**[Journey 9 — Team, Cloud & CI](docs/reference/09-team-cloud-ci.md)**
</td>
<td width="50%" valign="top">
### 🎚️ Tune & govern
*"Make it behave exactly how we want."*
```bash
lean-ctx compression standard
lean-ctx harden # enforce token discipline
```
Compression levels, tool profiles, themes, and rules governance.
**[Journey 10 — Customization & Governance](docs/reference/10-customization-and-governance.md)**
</td>
</tr>
<tr>
<td width="50%" valign="top">
### 📊 Prove the payoff
*"Show me the numbers."*
```bash
lean-ctx gain --deep # savings, cost, per-agent, heatmap
lean-ctx wrapped # shareable recap (also: gain --svg / gain --share)
lean-ctx savings # verified per-event ledger (auditable; savings verify)
```
All analytics live in the CLI/dashboard — never burning agent tokens.
**[Journey 11 — Analytics & Insights](docs/reference/11-analytics-and-insights.md)**
</td>
<td width="50%" valign="top">
### 📚 The full reference
*"I want to read everything."*
Every command and all 81 MCP tools, organized as user journeys, plus
appendices for the [CLI map](docs/reference/appendix-cli-map.md),
[MCP tools](docs/reference/appendix-mcp-tools.md), and
[paths & config](docs/reference/appendix-paths-and-config.md).
**[Reference index](docs/reference/README.md)**
</td>
</tr>
</table>
## Supported IDEs & AI tools
LeanCTX is a standard **MCP server**, so it works with any MCP-compatible client. Two integration modes are auto-selected per agent:
| Mode | How it works | Best for |
|---|---|---|
| **Hybrid** | MCP for cached reads (~13 tokens) + shell hooks for command compression | Agents with shell access (Cursor, Claude Code, Codex, ...) |
| **MCP** | All 80 tools via MCP protocol, no shell hooks | Protocol-only agents (JetBrains, VS Code, Zed, ...) |
### Agent compatibility matrix
| Agent | Hybrid | MCP | Setup |
|---|:---:|:---:|---|
| Cursor | ● | | `lean-ctx init --agent cursor` |
| Claude Code | ● | | `lean-ctx init --agent claude` |
| CodeBuddy | ● | | `lean-ctx init --agent codebuddy` |
| Augment CLI / VS Code | ● | | `lean-ctx init --agent augment` |
| Codex CLI | ● | | `lean-ctx init --agent codex` |
| Gemini CLI | ● | | `lean-ctx init --agent gemini` |
| Windsurf | ● | | `lean-ctx init --agent windsurf` |
| GitHub Copilot | ● | | `lean-ctx init --agent copilot` |
| CRUSH | ● | | `lean-ctx init --agent crush` |
| Hermes | ● | | `lean-ctx init --agent hermes` |
| OpenCode | ● | | `lean-ctx init --agent opencode` |
| Pi | ● | | `lean-ctx init --agent pi` |
| Qoder | ● | | `lean-ctx init --agent qoder` |
| Amp | ● | | `lean-ctx init --agent amp` |
| Cline | ● | | `lean-ctx init --agent cline` |
| Roo Code | ● | | `lean-ctx init --agent roo` |
| Kiro | ● | | `lean-ctx init --agent kiro` |
| Antigravity | ● | | `lean-ctx init --agent antigravity` |
| Amazon Q | ● | | `lean-ctx init --agent amazonq` |
| Qwen | ● | | `lean-ctx init --agent qwen` |
| Trae | ● | | `lean-ctx init --agent trae` |
| Verdent | ● | | `lean-ctx init --agent verdent` |
| Aider | | ● | `lean-ctx init --agent aider` |
| Continue | | ● | `lean-ctx init --agent continue` |
| JetBrains IDEs | | ● | `lean-ctx init --agent jetbrains` |
| QoderWork | | ● | `lean-ctx init --agent qoderwork` |
| VS Code | | ● | `lean-ctx init --agent vscode` |
| Zed | | ● | `lean-ctx init --agent zed` |
| Neovim | | ● | `lean-ctx init --agent neovim` |
| Emacs | | ● | `lean-ctx init --agent emacs` |
| Sublime Text | | ● | `lean-ctx init --agent sublime` |
> **Any MCP-compatible client** works out of the box — the table above shows agents with first-class auto-setup.
### When to use (and when not to)
**Great fit if you...**
- use AI coding tools daily and your sessions are shell-heavy (git/tests/builds)
- work in medium/large repos (50+ files / monorepos)
- want a local-first layer with **no telemetry by default**
**Skip it if you...**
- mostly work in tiny repos and rarely call the shell from your AI tool
- always need raw/unfiltered logs (you can still use `--raw`, but ROI is lower)
The honest fine print: the payoff depends on three levers — **reach** (own the
window via the proxy/engine, not just the `ctx_*` tool layer), **context
lifetime** (one long-lived session vs. a fresh process per phase), and
**provider pricing** (prompt-cache-priced vs. re-billed every turn). They stack
into a clear win where they line up and net to **break-even** where they don't.
See the [win vs. break-even matrix](docs/reference/14-performance-tuning.md#win-vs-break-even-at-a-glance)
for the full breakdown and how to tune for each case.
<a id="demo"></a>
## Demo
Try these in any repo:
```bash
lean-ctx read rust/src/server/mod.rs -m map
lean-ctx -c "git log -n 5 --oneline"
lean-ctx gain --live
lean-ctx dashboard # Context Manager (browser)
lean-ctx watch # TUI monitor
lean-ctx benchmark report .
```
- The repo ships the exact tapes used to render the GIFs in `demo/`
- Regenerate locally:
```bash
vhs demo/leanctx.tape
vhs demo/gain.tape
vhs demo/benchmark.tape
```
<a id="benchmarks"></a>
## Benchmarks
Real, reproduced numbers — never estimated. Measured on this repo with the GPT-4o
tokenizer (`o200k_base`); a tool that isn't installed is reported as such, never
guessed.
| Read mode | Compression | Tokens (50 files) | Quality |
|---|---:|---:|---:|
| Raw read | 0% | 533.2K | 100% |
| `map` | **98.1%** | 8.0K | 78% |
| `signatures` | **96.7%** | 14.0K | 96% |
| Cached re-read | ~99.99% | ~13 tok | 100% |
lean-ctx's **own cost is measured too**: the fixed per-session footprint it
injects (advertised tool schemas + MCP instructions) is ~2.1K tokens and
CI-gated via `lean-ctx doctor overhead --gate`, so it can only shrink. And the
long-lived proxy rail has a deterministic self-verify —
`lean-ctx benchmark dual-arm --json` replays a 15-turn session and prices it per
model (digest `f5ed145e61ce3689`, 99.4% input-side saving on cache-priced rails;
methodology: [bench/agent-task/r2](bench/agent-task/r2/README.md)).
Accuracy isn't a vibe: the lossy stages are **CI-gated**. A model-free A/B gate
proves the JSON crusher keeps *every* gold answer while cutting tokens, and proxy
rewrites are byte-stable by contract, so Anthropic (90%) / OpenAI (50%) prompt-cache
discounts survive compression. A deterministic **off-vs-on testbench**
(`lean-ctx eval testbench`) extends the proof to *answers*: it runs pinned real repos
through a raw-dump baseline and through lean-ctx at an identical token budget, grades
free-form QA with an LLM judge and code with each repo's own tests, and emits
`FINDINGS.md` (tokens / turns / walltime / quality) plus a regressions file — with a
committed recorded subset that blocks CI on any regression.
- **Latest snapshot**: [BENCHMARKS.md](BENCHMARKS.md)
- **Reproduce**: `lean-ctx benchmark report .`
## By the numbers
- **3,000+ GitHub stars** — and counting
- **280+ forks** — active community contributions
- **200+ releases** — shipped near-daily since launch
- **30+ supported AI coding agents** — broadest MCP compatibility
- **81 MCP tools** — from simple file reads to multi-agent orchestration
- Used in production by teams running Claude Code, Cursor, and Codex daily
- **Live adoption metrics**: [leanctx.com/metrics](https://leanctx.com/metrics/) — installs, stars and savings, updated continuously
## Docs
- **Reference (every function, by user journey)**: [docs/reference/](docs/reference/README.md) — 11 journeys + CLI/MCP/config appendices
- **For AI agents / LLMs**: [llms.txt](llms.txt) — a curated, machine-readable map of lean-ctx (per the [llms.txt](https://llmstxt.org) convention)
- Getting started: https://leanctx.com/docs/getting-started
- Tools reference: https://leanctx.com/docs/tools/
- CLI reference: https://leanctx.com/docs/cli-reference/
- What is LeanCTX: https://leanctx.com/what-is-leanctx/
- Comparison (vs RTK, Context+, MemGPT): https://leanctx.com/compare/
- Pricing & Cloud (local use is free forever): https://leanctx.com/pricing/
- FAQ: [discord-faq.md](discord-faq.md)
- Feature catalog (SSOT snapshot): [LEANCTX_FEATURE_CATALOG.md](LEANCTX_FEATURE_CATALOG.md)
- Monorepo guide: [docs/guides/monorepo.md](docs/guides/monorepo.md)
- Architecture: [ARCHITECTURE.md](ARCHITECTURE.md)
- Vision: [VISION.md](VISION.md)
## Privacy & security
- **No telemetry by default**
- **Optional anonymous stats sharing** (opt-in during setup)
- **Disableable update check** (config `update_check_disabled = true` or `LEAN_CTX_NO_UPDATE_CHECK=1`)
- **40+ security hardening fixes** in v3.5.16 (path traversal, injection, CSPRNG, CSP, resource limits — [details](CHANGELOG.md))
- **Context Governance Benchmark self-assessment**: graded **C2 — Managed** against the 32-control [CGB v1.0-draft](https://github.com/yvgude/context-governance-benchmark) spec, gaps declared — [docs/compliance/cgb-self-assessment.md](docs/compliance/cgb-self-assessment.md)
- Runs locally; your code never leaves your machine unless you explicitly enable cloud sync
See [SECURITY.md](SECURITY.md).
## Uninstall
One command removes **everything** — it stops all processes, then deletes hooks,
editor configs, rules, autostart (LaunchAgent/systemd), the data dir, **and the
binary itself**:
```bash
lean-ctx uninstall # full clean removal
lean-ctx uninstall --dry-run # preview every change, write nothing
lean-ctx uninstall --keep-config # keep MCP configs + rules (for reinstall)
lean-ctx-off # or just disable for the current shell session
```
No binary on PATH (or you used the curl installer)? Run the same removal from the installer:
```bash
curl -fsSL https://leanctx.com/install.sh | sh -s -- --uninstall
```
If you installed via a package manager, `uninstall` removes everything it wrote and
tells you the one command to finish removing the binary:
```bash
brew uninstall lean-ctx # Homebrew
cargo uninstall lean-ctx # cargo install
npm uninstall -g lean-ctx-bin # npm
pi uninstall npm:pi-lean-ctx # Pi Coding Agent
```
## Star History
<a href="https://star-history.com/#yvgude/lean-ctx&Date">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=yvgude/lean-ctx&type=Date&theme=dark" />
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=yvgude/lean-ctx&type=Date" />
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=yvgude/lean-ctx&type=Date" />
</picture>
</a>
## Contributing
Start with [CONTRIBUTING.md](CONTRIBUTING.md). Easy first PR: propose a new CLI compression pattern via the [issue template](.github/ISSUE_TEMPLATE/compression_pattern.md).
## License
Apache License 2.0 — see [LICENSE](LICENSE).
--- lean-ctx: ctx_compose bundles search+read+symbols in one call ---
+7
View File
@@ -0,0 +1,7 @@
# WeHub 来源说明
- 原始项目:`yvgude/lean-ctx`
- 原始仓库:https://github.com/yvgude/lean-ctx
- 导入方式:上游默认分支的最新快照
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
- 本文件仅用于记录来源,不代表 WeHub 是原项目作者
+370
View File
@@ -0,0 +1,370 @@
# Security Policy
## Reporting a Vulnerability
If you discover a security vulnerability in lean-ctx, please report it privately:
- **Email**: yves@pounce.ch
- **GitHub**: [Create a private security advisory](https://github.com/yvgude/lean-ctx/security/advisories/new)
- **Response time**: We aim to acknowledge reports within 48 hours
- **Disclosure**: We follow responsible disclosure practices (90-day embargo)
**Please do NOT:**
- Open public GitHub issues for security vulnerabilities
- Disclose vulnerabilities on social media or forums before we've had a chance to address them
---
## What lean-ctx Does (and Doesn't Do)
lean-ctx is a **local-only CLI tool and MCP server**. Understanding its scope helps assess risk:
**Does:**
- Read files from your local filesystem (explicit reads and tool-driven scans within the project boundary)
- Execute shell commands (only commands you or your AI tool explicitly invoke)
- Cache file contents in memory during a session
- Store statistics in `~/.lean-ctx/stats.json` (command counts, token savings)
- Store session state in `~/.lean-ctx/sessions/` (task context, findings)
### I/O Boundary (PathJail + Roles)
lean-ctx enforces a **project boundary** for filesystem I/O:
- **PathJail**: all tool path inputs are resolved and jailed under the current `project_root`.
- If a path would escape, the call fails with a clear hint to explicitly allow additional roots.
- **Explicit allow roots**:
- Env: `LEAN_CTX_ALLOW_PATH` (or `LCTX_ALLOW_PATH`) — a path list (`:` on Unix, `;` on Windows)
- Config: `allow_paths` in `~/.lean-ctx/config.toml` (whitelist only); `extra_roots` (whitelist + multi-root scanning)
- `~`, `$VAR` and `${VAR}` are expanded in these entries (no shell runs for config files)
- **Disabling the jail** (sandboxed environments where the OS is the boundary):
- Config: `path_jail = false` in `~/.lean-ctx/config.toml`
- Compile-time: the `no-jail` cargo feature
- The legacy `LEAN_CTX_NO_JAIL=1` env var was removed in v3.7.4 and has no effect
- Full reference: `docs/reference/appendix-paths-and-config.md` §5; `lean-ctx doctor` reports the effective jail state
- **Symlink escape protection**: canonicalization ensures that symlinks pointing outside the jail are rejected.
In addition, roles can restrict **unsafe I/O**:
- **Secret-like deny-by-default**:
- Search skips secret-like files (e.g. `.env`, `*.pem`, `id_rsa`, `.ssh/`, `.aws/`) unless the active role explicitly allows them.
- Artifact registry resolution rejects secret-like artifact paths unless allowed (artifacts are indexed/shareable by design).
- Direct reads/edits can warn or error depending on boundary mode.
- **`.gitignore` bypass is policy-gated**:
- `ctx_search ignore_gitignore=true` requires explicit role permission (typically the `admin` role).
- **Boundary mode**:
- Roles can set `io.boundary_mode = "warn" | "enforce"`.
- Env override: `LEAN_CTX_IO_BOUNDARY_MODE=warn|enforce`.
- **Auditability**:
- Boundary denials/warnings emit local `PolicyViolation` events (no secret content is returned as part of the violation).
### Threat Model (v2)
**Primary risks (local-only, but high impact):**
- **Accidental secret exfiltration to LLMs** via `ctx_read`, `ctx_search`, compressed `ctx_shell`, archives, or exported artifacts.
- **Boundary escapes** via absolute paths, symlinks, linked projects, or artifact path tricks.
- **Amplification / token burn** by scanning large files or returning unbounded outputs.
- **ReDoS** via user-supplied regex patterns in `ctx_search`.
- **Cross-workspace data access** in team server deployments (IDOR).
**Core mitigations:**
- **PathJail** + explicit allow roots (`LEAN_CTX_ALLOW_PATH` / `allow_paths`).
- **Role-gated unsafe I/O** (`ignore_gitignore`, secret-like allow).
- **Secret path check on all MCP read paths** — `.env`, SSH keys, etc. blocked by default.
- **Shell CWD jail enforcement** — explicit `cwd` parameters are jail-checked, `cd` targets validated.
- **Deterministic redaction** on tool outputs (non-admin roles, and for persisted archives).
- **Hard caps** on reads and outputs to limit DoS/token burn.
- **Regex guards** — pattern length (1024 chars) and DFA size (1 MiB) limits on `ctx_search`.
- **MCP message size limit** — 32 MiB cap on JSON-RPC message size.
- **Constant-time token comparison** in all auth paths (dashboard, HTTP server, team server).
- **Team server tenant isolation** — workspace enforced from authenticated token, not query parameters.
- **JSON-RPC batch rejection** — batch requests rejected on team server to prevent scope bypass.
- **Event payload redaction** — REST API responses redacted to `Summary` level by default.
- **Pipeline archive redaction** — shell output archives redacted before storage.
- **UDS socket permissions** — `0o600` enforced on Unix domain sockets after bind.
- **Error response sanitization** — internal details logged server-side, generic codes returned to clients.
**Optional network activity (fully disableable):**
- **Update check**: a lightweight daily GET to `leanctx.com/version.txt` to notify you of new versions. Sends only the current version as User-Agent. Disable with `update_check_disabled = true` in `~/.lean-ctx/config.toml` or `LEAN_CTX_NO_UPDATE_CHECK=1`.
- **Anonymous stats sharing** (opt-in, off by default): if you enable `contribute_enabled` in setup, anonymized compression statistics (token counts, compression ratios — no file names, no code, no PII) are periodically sent to `api.leanctx.com`.
**Does NOT:**
- Collect tracking analytics, fingerprints, or PII
- Access files outside of requested paths
- Store or transmit credentials, API keys, or secrets
- Require elevated privileges (runs as your user)
---
## Automated Security Checks
Every push and pull request triggers our CI security pipeline:
1. **`cargo audit`** — Scans dependencies for known CVEs
2. **`cargo clippy`** — Enforces Rust safety lints (warnings = errors)
3. **Dangerous pattern scan** — Detects potentially unsafe code patterns:
- Shell injection vectors (`Command::new("sh")` with user input)
- Network operations (`reqwest::`, `std::net::`, `hyper::`)
- Unsafe code blocks (`unsafe {`)
- Environment manipulation (`.env("LD_PRELOAD")`)
- Hardcoded secrets or obfuscated strings
4. **`cargo test`** — Full test suite must pass
---
## Critical Files Requiring Enhanced Review
Changes to these files receive extra scrutiny:
| File | Risk | Why |
|------|------|-----|
| `rust/src/shell/` | Shell execution | Wraps your shell, executes commands |
| `rust/src/server/` | MCP protocol | Handles all tool calls from AI editors/agents |
| `rust/src/hooks/` | Editor integration | Installs hooks/config into Claude Code, CodeBuddy, Cursor, etc. |
| `rust/src/core/cache.rs` | File caching | Reads and stores file contents |
| `rust/Cargo.toml` | Supply chain | Dependency manifest |
| `.github/workflows/*.yml` | CI/CD | Release pipeline integrity |
---
## Dependency Security
All dependencies in `Cargo.toml` meet these criteria:
- **Established crates**: All 29 dependencies are well-known, widely-used Rust crates
- **License**: Apache-2.0 compatible
- **Active maintenance**: Recent commits within 6 months
- **Minimal network**: `ureq` (lightweight HTTP client) used only for version check and opt-in cloud sync
Key dependencies and their purpose:
| Crate | Purpose | Downloads |
|-------|---------|-----------|
| `rmcp` | MCP protocol (stdio transport only) | Rust MCP reference impl |
| `tiktoken-rs` | Token counting (o200k_base) | OpenAI's tokenizer |
| `tree-sitter` + grammars | AST parsing for 26 languages | Mozilla's parser |
| `tokio` | Async runtime (for MCP server) | 200M+ downloads |
| `serde` / `serde_json` | JSON serialization | 400M+ downloads |
| `similar` | Myers diff algorithm | Well-established |
| `walkdir` | Directory traversal | 100M+ downloads |
---
## VirusTotal False Positives
Rust binaries are frequently flagged by ML-based antivirus engines (particularly Microsoft Defender's `Wacatac.B!ml` classifier). This is a **known issue** affecting many Rust projects:
- [Rust lang discussion on false positives](https://users.rust-lang.org/t/rust-programs-flagged-as-malware/49799)
- 1/72 engines flagging = definitively a false positive
- The `!ml` suffix in `Wacatac.B!ml` means "Machine Learning detection" (heuristic, not signature-based)
**Why it happens:**
- Statically linked binaries (~30 MB) are unusual for Windows
- `strip = true` + `lto = true` optimizations alter binary structure
- New/unsigned executables trigger ML classifiers trained on known-good signed software
**How to verify lean-ctx yourself:**
1. Build from source: `cargo install lean-ctx` (compiles on your machine)
2. Compare SHA256 checksums against our [GitHub Releases](https://github.com/yvgude/lean-ctx/releases)
3. Audit the source code: the entire codebase is open source
---
## Build Reproducibility
To verify that a release binary matches the source code:
```bash
# Clone and build
git clone https://github.com/yvgude/lean-ctx.git
cd lean-ctx/rust
cargo build --release
# Compare with installed version
lean-ctx --version
./target/release/lean-ctx --version
```
SHA256 checksums for all release binaries are published in each [GitHub Release](https://github.com/yvgude/lean-ctx/releases).
---
## Disclosure Timeline
When vulnerabilities are reported:
1. **Day 0**: Acknowledgment sent to reporter
2. **Day 7**: Severity assessment completed
3. **Day 14**: Patch development begins
4. **Day 30**: Patch released + CVE filed (if applicable)
5. **Day 90**: Public disclosure
Critical vulnerabilities (RCE, data exfiltration) are fast-tracked.
---
## Known Residual Risks
### TOCTOU (Time-of-Check to Time-of-Use)
**Status:** Mitigated on Unix; residual risk on Windows.
A race window exists between `jail_path` validation and the subsequent file operation. Mitigations in place: standard reads open with `O_NOFOLLOW` (Unix) and reject symlinks; `ctx_edit` additionally rejects symlinks on both its read and write paths (lstat pre-check on all platforms, `O_NOFOLLOW` on Unix) and re-verifies the file fingerprint (size/mtime/md5) immediately before writing. On Windows there is no `O_NOFOLLOW` equivalent, so the lstat pre-check is the only guard — it rejects symlinks **and all NTFS reparse points (junctions, mount points)** via `pathutil::is_symlink_or_reparse`, and `read_file_nofollow` applies the same lstat check before opening. The residual risk is the unavoidable check→open race window.
**Recommendation for regulated environments:** Run lean-ctx inside a container or VM where the filesystem is controlled and no untrusted processes can modify symlinks concurrently.
### ctx_execute Sandbox Naming
**Status:** Documented limitation.
The `ctx_execute` tool provides **timeout enforcement** and **output capping** but does **not** provide OS-level sandboxing (no containers, namespaces, or seccomp filters). The term "sandbox" in tool descriptions refers to the execution boundary, not kernel-level isolation.
**Recommendation for regulated environments:** Disable `ctx_execute` via role configuration (`denied: ["ctx_execute"]`) or run lean-ctx in a pre-existing container sandbox.
### Shell Command Validation (REQ-57177, GH #391)
**Status:** Defense in depth — the agent's permission model remains the primary boundary. **`ctx_shell` is not a sandbox.**
`ctx_shell` and `lean-ctx -c` enforce, in both allowlist and blocklist-only mode:
- a deny-by-default executable allowlist when configured (AST-segmented: every segment of a pipeline/compound command must be allowed),
- `eval`/`exec`/`source` unconditionally blocked; `$()`/backticks blocked at command position,
- **interpreter inline-code blocking**: `bash -c`, `sh -c`, `python -c`, `node -e`, … are rejected (including via delegation wrappers like `env`, `timeout`, `xargs`) — quoting a payload inside `bash -c '…'` does not bypass the file-write or allowlist checks because the interpreter call itself is refused,
- file-write detection (`>`, `>>`, `tee`, heredoc-to-file, `dd of=`, `curl -o/-O`, `wget` to file) — writes belong to the editor's native Write/Edit tools where the agent's permission UI governs them,
- dangerous-flag blocking (`git --upload-pack`, `tar --to-command`, `find -exec`, `awk system()`), inline env hijack blocking (`PATH=`, `LD_PRELOAD=`, `GIT_SSH_COMMAND=`, …), and dangerous env-key filtering on the MCP `env` parameter.
Enforcement applies to the MCP path, hook-child mode and every non-interactive CLI invocation; an interactive human terminal gets a warning instead (`LEAN_CTX_ALLOWLIST_WARN_ONLY=1` opts out explicitly). Cloud/infra mutation CLIs (terraform, kubectl, aws, …) are excluded from the default allowlist and require per-tool opt-in (`lean-ctx allow <cmd>`). `shell_strict_mode = true` upgrades the warn-only heuristics (command substitution in arguments, pipe-to-bare-interpreter) to hard blocks.
**Explicitly out of scope:** commands run with the invoking user's full privileges. Anything the user can do, an allowed command can do — `cp`/`rsync` can copy any file the OS lets the user read, package managers execute arbitrary install scripts, `npm test` runs project code. A blocklist cannot enumerate these; pretending otherwise would be security theater.
**Rationale:** Shell filters can be bypassed by a sufficiently creative attacker, so the agent's permission model (Claude Code allowlists, Cursor approval dialogs) remains the primary boundary — lean-ctx's allowlist is a second, independent layer, not a replacement.
**Mitigation for untrusted agents:** Use role-based `denied: ["ctx_shell"]` to disable shell access entirely, enable the deny-by-default allowlist with a minimal command set, and/or run the whole agent stack inside an OS sandbox (container, gVisor/Firecracker, bwrap, seccomp/AppArmor) — that is the correct layer for kernel-grade isolation.
### PathJail TOCTOU Race (REQ-57178)
**Status:** Mitigated on Unix; residual risk on Windows.
A race window exists between `jail_path` validation and the subsequent file operation. Mitigations: symlink-following canonicalization before access, `O_NOFOLLOW` + symlink rejection on read paths (Unix), and symlink rejection on `ctx_edit` write paths (all platforms, lstat-based; on Windows this includes NTFS junctions and other reparse points). Home-level IDE config dirs (`~/.cursor`, `~/.claude`, …) are excluded from the jail's allow-list by default (`allow_ide_config_dirs` opts in).
**Windows file permissions:** the Unix `0o600`/`0o700` tightening (cloud credentials, crash log) has no direct Windows equivalent; protection relies on the default NTFS ACL of the user profile (`%USERPROFILE%` is not readable by other non-admin users). Machines with custom ACLs on the profile directory should verify `%USERPROFILE%\.lean-ctx` inherits owner-only access.
**Mitigation:** For regulated environments: run lean-ctx inside a container where no untrusted processes can modify symlinks concurrently.
### Cloud Server Database TLS (REQ-57188)
**Status:** Accepted risk — localhost-only by default.
The cloud server's PostgreSQL connection does not enforce TLS by default. This is acceptable because the cloud server is designed for localhost/loopback deployment where DB traffic does not traverse a network.
**Mitigation for production:** Set `DATABASE_URL` with `?sslmode=require` or use a connection string that enforces TLS. When deployed behind a reverse proxy (nginx/Caddy), ensure TLS terminates before the DB.
### HuggingFace Model Downloads
**Status:** Documented risk.
Embedding models for semantic search are downloaded from HuggingFace Hub. Verification is size-based heuristic only, not cryptographic (no SHA256 pinning for model files).
**Recommendation for regulated environments:** Pre-provision models manually from an internal mirror with signature verification. Set `LEAN_CTX_EMBEDDING_MODEL_DIR` to point to the pre-provisioned directory to skip downloads entirely.
### Project-Scope Config Influences Injected Context (external audit, finding 4)
**Status:** Mitigated by Workspace Trust (selective gating + content-hash pin).
A cloned repository's `.lean-ctx.toml` is merged over the global config by `Config::merge_local`. That merge can raise **security-sensitive** settings — replace the shell allowlist, widen the path jail (`allow_paths` / `extra_roots`), repoint the proxy upstream, define command aliases, change `rules_scope` / `rules_injection`, or widen the exposed tool surface (`tool_profile` / `tools_enabled` / `default_tool_categories`). Opening an untrusted clone with an agent would let the repo silently weaken lean-ctx's own boundaries.
**Mitigation (shipped):** lean-ctx now applies a VS-Code-style **Workspace Trust** gate. For a workspace the user has not trusted, the security-sensitive overrides above are **withheld** (comfort knobs like `compression`/`theme` still apply) and a `[SECURITY]` warning is logged; `lean-ctx doctor` shows the state. Trust is granted explicitly with `lean-ctx trust` and pinned to BOTH the workspace path AND a content hash of `.lean-ctx.toml`, so editing the file after trust re-gates it (a "trust once, modify later" change cannot take effect silently). Headless/fleet use can opt in via `LEAN_CTX_TRUST_WORKSPACE=1` or `LEAN_CTX_TRUSTED_ROOTS`.
**Residual:** Model-visible *content* lean-ctx injects (the static `<!-- lean-ctx -->` rules block, hook `additionalContext`, `[VERIFY]`/`[HINT]` suffixes) is itself auditable and not repo-controlled. Review a clone's `.lean-ctx.toml` before `lean-ctx trust`.
### LLM Proxy is a Full MITM on API Traffic (external audit, finding 6)
**Status:** By design; loopback-bound and disabled by default.
When enabled, the optional LLM proxy (`lean-ctx proxy`) reads and rewrites every request body (compression, history pruning) and sees `Authorization` headers in cleartext — a concentrated sensitive-data surface. Any process able to reach the port, or a forwarding bug, would expose prompts, completions and API keys.
**Mitigation:** The proxy is disabled by default and binds to loopback (`127.0.0.1`) with an auto-generated auth token when enabled; keep it loopback-bound. The MCP `ctx_*` tools deliver compression savings without routing API traffic through any proxy — leave it disabled unless you need pay-as-you-go key forwarding.
---
## Security Architecture for Enterprise Deployments
### Recommended Configuration (Bank / Regulated)
LeanCTX splits **global settings** (`config.toml`) from **capability policy**
(role files): `config.toml` does *not* define roles or a top-level `[io]`
policy — per-session I/O limits live on the active role, which is selected via
`LEAN_CTX_ROLE`.
**1. Global hardening — `~/.lean-ctx/config.toml`:**
```toml
update_check_disabled = true # no daily update check
path_jail = true # keep the filesystem jail on (default)
[cloud]
contribute_enabled = false # no anonymous stats sharing (default)
```
**2. A locked-down role — `~/.lean-ctx/roles/bank.toml`:**
```toml
[role]
name = "bank"
description = "Locked-down role for regulated environments"
[tools]
denied = ["ctx_execute", "ctx_shell"] # no code or shell execution
[io]
boundary_mode = "enforce" # error (not warn) on boundary hits
allow_secret_paths = false # never read .env / keys
allow_ignore_gitignore = false # never scan .gitignore'd paths
allow_cross_project_search = false # stay within this project
```
**3. Activate the role** where lean-ctx (or the MCP server) starts. The active
role resolves in order **env → project `.lean-ctx/roles/` → global
`~/.lean-ctx/roles/` → built-in**:
```bash
export LEAN_CTX_ROLE=bank
```
### Network Surface
| Endpoint | Purpose | Disable |
|----------|---------|---------|
| `leanctx.com/version.txt` | Update check (daily GET) | `update_check_disabled = true` |
| `api.leanctx.com` | Opt-in anonymous stats | `contribute_enabled = false` (default) |
| `huggingface.co` | Embedding model download | Pre-provision models, set `LEAN_CTX_EMBEDDING_MODEL_DIR` |
| `localhost:PORT` | Dashboard (local TCP) | Don't start dashboard, or bind to loopback only |
| UDS socket | Daemon IPC | Permissions `0o600`, owner-only access |
### Team Server Hardening
When running the team server (`lean-ctx team-server`):
1. **Token rotation**: Rotate workspace tokens periodically. Tokens are stored in the team config.
2. **Scope minimization**: Grant only necessary scopes per workspace token (e.g., `read` only, no `shell`).
3. **Network isolation**: Bind the team server to an internal network interface, not `0.0.0.0`.
4. **Audit log monitoring**: Team server writes audit logs for all tool calls. Monitor for denied requests.
5. **JSON-RPC batch requests**: Rejected by default to prevent scope bypass.
### Supply Chain
- **`cargo audit`** runs on every CI push (zero known CVEs tolerated).
- **`cargo deny`** checks licenses and advisories.
- **npm `postinstall.js`** verifies SHA256 of downloaded binaries against `SHA256SUMS` release asset.
- **GitHub Actions** uses pinned action versions with hash verification.
---
## Contact
- **Security issues**: yves@pounce.ch
- **General questions**: [GitHub Discussions](https://github.com/yvgude/lean-ctx/discussions)
- **Discord**: [Join our server](https://discord.gg/pTHkG9Hew9)
---
**Last updated**: 2026-06-21
+111
View File
@@ -0,0 +1,111 @@
# LeanCTX Vision
> **Control what your AI can see.**
>
> Ecosystem overview: [`ECOSYSTEM.md`](ECOSYSTEM.md)
## The Cognitive Context Layer
High performance with LLMs isn't about bigger context windows — it's about
**information density**. LeanCTX is the cognitive context layer between your
AI and your code: every token reaching the LLM carries maximum signal, and
every byte of noise stripped away is a byte of reasoning gained.
> The winners won't be those who can afford 1M-token contexts.
> They'll be those who achieve the same result with 10K.
## The four dimensions
1. **Compression layer (input efficiency)** — AST-based signatures, delta
loading, session caching (re-reads ~13 tokens), entropy filtering, 95+ CLI
compression patterns, 26 tree-sitter languages, 10 read modes.
2. **Semantic router (model selection)** — intent detection, mode prediction
learned per file type, LITM-aware positioning per model family.
3. **Context manager (memory architecture)** — Context Continuity Protocol
(~400 tokens instead of ~50K cold start), context ledger, multi-agent
coordination, temporal knowledge system, property graph with hybrid
search fusion.
4. **Quality guardrail (output verification)** — compression safety levels,
deterministic anchoring, 19 versioned contracts with CI drift gates,
policy packs, tamper-evident audit trails, Ed25519-signed evidence bundles.
Technical depth: [`docs/cognition-interface.md`](docs/cognition-interface.md) ·
[`CONTRACTS.md`](CONTRACTS.md)
## Two halves of context, one pipeline
Getting the right knowledge into the window is really *two* problems, and most
tools only solve one:
- **Compress what fits.** A file, a diff, a shell log, a handful of docs — the
right move is to fit it into the window *losslessly* (read modes, structural
crushing, cached re-reads). Embedding-and-retrieving here throws away
information you already had room for.
- **Retrieve what doesn't.** A large or dynamic knowledge base has to be
retrieved — and lean-ctx does it with a *hybrid* retriever: lexical BM25 +
learned-sparse SPLADE + dense vectors, fused with Reciprocal Rank Fusion and
reranked, never a single cosine signal. Embeddings run from a **local ONNX
model** (swappable; a model2vec fast path skips the attention pass), so recall
is strong without an external vector DB, an embedding API, or a minutes-long,
CPU-melting index build.
The failure mode of naive RAG is applying *retrieve* to everything, including
material that never needed it — more chunks, less signal, quiet drift. lean-ctx
runs both halves under **one pipeline** and picks the right one for the material.
**The moat is structure.** A codebase is not a bag of paragraphs: functions call
functions, changes have a blast radius, symbols have definitions and references.
lean-ctx is structure-aware (tree-sitter AST + a code graph) and *uses* that
graph at retrieval time — associative spreading activation surfaces structurally
close code, and reranking grounded in 2025 code-retrieval research (CoRNStack,
SACL, SweRank) sharpens the top results. Retrieval is *precise on code* in a way
pure text-embedding search cannot be.
**Knowledge stays yours.** What the engine learns is portable, not harvested:
export it as open, git-diffable **OKF** Markdown (interop with any OKF reader, no
lock-in) or as a signed, versioned **`.ctxpkg`** for distribution — the same
snapshot rendered for reading or shipping. Portability is a property of the
format, not a paid feature. And when a team wants a heavier, external RAG across
many repos and document types, that plugs in as an **addon** rather than bloating
the always-local core.
## Principles
- **Local-first, zero telemetry.** Nothing leaves your machine automatically —
ever. The engine learns locally (read modes, compression thresholds,
bandits); what it learns belongs to you.
- **Learned optimization is portable, not harvested.** Tuned profiles can be
exported as signed `.ctxpkg` packages and shared through the registry — a
deliberate, inspectable file, not a background upload.
- **Evidence over claims.** Policy decides what an agent may see; signed
evidence proves what it saw. Compliance reports (EU AI Act, ISO/IEC 42001,
SOC 2) are generated from real session data, offline-verifiable.
- **One binary, 30+ tools.** Cursor, Claude Code, CodeBuddy, Windsurf, Copilot, Codex,
Gemini, JetBrains and more — the same engine everywhere.
## Direction
- **Context Time Machine** — the layer state (what the model saw, why, and at
what token ROI) is now a git-anchored, signed, navigable artifact: rewind to
any commit, reproduce it, resume from it, or share it. The temporal axis
through everything lean-ctx does — it *decides, remembers, guards, proves, and
now replays*. **Shipped:** the snapshot engine (`snapshot
create/list/show/verify`), dashboard replay, `restore [--git]`, and signed
file-based `publish`/`import`. **Next:** a `ctxpkg.com` registry for hosted,
versioned history and a side-by-side model-view git-diff replay. See
[`docs/concepts/context-time-machine.md`](docs/concepts/context-time-machine.md).
- **Context as Code** — declarative pipelines, profiles and policies in TOML,
version-controlled like infrastructure.
- **Cognition interface** — constraints-aware instruction compilation,
attention-aware layout, budget/SLO enforcement, proof-carrying context.
- **Unified context graph** — code, tests, commits, CI runs and knowledge in
one semantic graph with graph-aware reads.
- **Provider framework** — issues, tickets, CI and logs flowing through the
same consolidation pipeline as code.
- **Org-wide context** — agent handoffs, cross-session memory and team
accounts as the substrate for fleet-level context (see `ECOSYSTEM.md`).
The end state: an AI that sees only what matters, remembers what's relevant,
and reasons at maximum capacity — governed by policies you define.
**Tokens are the new gold. Context is the new infrastructure. Spend both wisely.**
Binary file not shown.

After

Width:  |  Height:  |  Size: 162 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 734 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 302 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 521 KiB

+6
View File
@@ -0,0 +1,6 @@
# Run artifacts stay local until a result is published deliberately.
runs/
.cache/
# Generated once, committed deliberately with the frozen protocol.
# (tasks.lock.json is tracked — do NOT ignore it.)
logs/
+14
View File
@@ -0,0 +1,14 @@
You are working in a git repository checked out at the commit where a bug
report was filed. Your job is to fix the issue described below.
<problem_statement>
{problem_statement}
</problem_statement>
Requirements:
- Fix the issue by editing the repository source code.
- Make the minimal change that resolves the issue.
- Do not modify, delete or add test files — the fix is judged by the
project's own tests, which run later.
- You may run commands (e.g. targeted tests) to understand and verify the fix.
- Leave your final changes saved in the working tree. Do not commit.
+93
View File
@@ -0,0 +1,93 @@
# Agent-Task-Benchmark v1 — Pre-Registered Protocol (GL #493)
Status: **FROZEN at commit time.** Changes after the first recorded run require a
numbered amendment (`A1`, `A2`, …) appended under *Amendments* — the original
text is never edited. This mirrors the standard set by the external tokbench
study (GH #361): protocol first, runs second, publication third.
## 1. Question
Does lean-ctx change the **outcome** of an agentic coding workload — task
success rate and cost per solved task — compared to the identical agent
without lean-ctx?
This deliberately measures outcome, not token arithmetic. Token deltas are
reported, but the primary endpoints are:
- **resolved rate** (per the official SWE-bench evaluation harness), and
- **cost per resolved task** (billed USD / resolved count).
## 2. Workload
- **Dataset:** `princeton-nlp/SWE-bench_Verified` (test split), the
human-validated 500-instance subset.
- **Subset size:** N = 15 instances.
- **Selection rule (deterministic, no cherry-picking):** sort all instances by
`instance_id` ascending; group by `repo`; visit repos in ascending name
order round-robin, taking the lexicographically first untaken instance from
each repo per cycle, until N instances are selected. The result is committed
as `tasks.lock.json` (content-hashed into the result artifact). The lock is
generated once by `select_tasks.py` and never regenerated for v1.
- Instances are independent; each run starts from a clean checkout of
`base_commit`.
## 3. Arms
Two arms, identical in every respect except lean-ctx:
| | `native` | `leanctx` |
|---|---|---|
| Agent | Claude Code headless (`claude -p`), pinned version recorded in `meta.json` | identical |
| Prompt | `PROMPT.md` template, identical text | identical |
| MCP config | none (`--strict-mcp-config` with empty config) | exactly the registration `lean-ctx init --agent claude` writes, extracted into an explicit config and pinned via `--strict-mcp-config`; missing registration aborts the run; lean-ctx version recorded |
| Rules file | none | `CLAUDE.md`/rules as written by `lean-ctx init` (stock; no hand-tuning) |
| HOME | fresh per-run temp HOME (no user-level config bleed) | identical |
| Max turns | 40 | identical |
| Model | the agent runtime's pinned default model, recorded per run | identical |
The fresh-HOME isolation exists because the operator's real machine has
lean-ctx globally installed; without it the `native` arm would be
contaminated.
## 4. Measurement
- **Patch:** after the agent exits, `git add -A && git diff --cached` in the
task workspace is the submitted `model_patch`.
- **Resolution:** official `swebench.harness.run_evaluation` (dockerized) on
the per-arm `predictions.jsonl`. An instance counts as resolved iff the
official report lists it as resolved. No manual judging.
- **Tokens & cost:** taken from the agent runtime's own final usage report
per run (stream-json `result` event: input/output/cache tokens,
`total_cost_usd`). We do not estimate; if the runtime reports no usage the
run is marked `usage_missing` and excluded from cost endpoints (counted in
resolution endpoints).
- **Wall time:** harness-measured per run.
## 5. Endpoints
Primary:
1. resolved-rate per arm (resolved / N),
2. cost per resolved task per arm (sum billed USD / resolved).
Secondary: billed input tokens per run (median), output tokens, wall time,
turns used, lean-ctx tool-call adoption in the `leanctx` arm (from
transcripts).
## 6. Honesty constraints
- Both arms run from the same task lock, same prompt, same limits.
- All transcripts (`transcript.jsonl`), patches and the evaluation report are
retained as raw artifacts and published alongside the result.
- Negative or neutral results are published unchanged.
- The result artifact embeds the SHA-256 of this protocol and of
`tasks.lock.json`; `report.py` emits the artifact digest for signing
(`ssh-keygen -Y sign`) so third parties can verify nothing moved after the
fact.
- Known limitation, stated up front: N=15 with one seed is a pilot-grade
sample — confidence intervals are wide; the claim is directional, not a
leaderboard. Provider-side prompt caching is active for both arms equally
(it is part of the product reality being measured).
## 7. Amendments
*(none yet)*
+78
View File
@@ -0,0 +1,78 @@
# Agent-Task-Benchmark v1 (GL #493)
Outcome evidence, not token arithmetic: **does lean-ctx change task success
rate and cost per solved task** for an agentic coding workload? Two identical
arms (Claude Code headless, native vs. lean-ctx MCP) over a deterministic
SWE-bench-Verified subset, judged by the official SWE-bench evaluation.
The methodology is pre-registered in [`PROTOCOL.md`](PROTOCOL.md) — read it
first; it is frozen, and changes require numbered amendments. Negative or
neutral results are published unchanged.
## Prerequisites
- Python ≥ 3.9, `pip install -r requirements.txt`
- `claude` (Claude Code CLI) on PATH, `ANTHROPIC_API_KEY` exported
- `lean-ctx` on PATH (pinned release; version is recorded per run)
- Docker (only for the evaluation step)
- Disk: repo mirrors (~2 GB) + SWE-bench evaluation images
> Safety: agent runs execute model-chosen shell commands with permissions
> skipped (standard for SWE-bench harnesses). Run on a disposable machine or
> container, not on a workstation with credentials you care about.
## Runbook
```bash
cd bench/agent-task
# 0a. Preflight — verify binaries, auth, network, disk BEFORE spending money
python3 -m swebench_harness.preflight # add --evaluate before step 3
# 0b. Task set: already frozen as tasks.lock.json (n=15, 12 repos) — do NOT
# regenerate for v1 (select_tasks refuses if the lock exists).
# 1. Smoke (1 instance, both arms) — validates plumbing end to end
python3 -m swebench_harness.run_arm --arm native --run-id smoke --instance <id-from-lock>
python3 -m swebench_harness.run_arm --arm leanctx --run-id smoke --instance <id-from-lock>
# 2. Full run (N=15 × 2 arms; resumable — finished runs are skipped)
python3 -m swebench_harness.run_arm --arm native --run-id v1
python3 -m swebench_harness.run_arm --arm leanctx --run-id v1
# 3. Predictions + official evaluation (docker)
python3 -m swebench_harness.collect --run-id v1
python3 -m swebench.harness.run_evaluation \
--dataset_name princeton-nlp/SWE-bench_Verified \
--predictions_path runs/v1/predictions-native.jsonl --run_id v1-native --max_workers 4
python3 -m swebench.harness.run_evaluation \
--dataset_name princeton-nlp/SWE-bench_Verified \
--predictions_path runs/v1/predictions-leanctx.jsonl --run_id v1-leanctx --max_workers 4
# 4. Endpoints + verifiable artifact (+ optional signature)
python3 -m swebench_harness.report --run-id v1 \
--eval-report native=claude-code-native.v1-native.json \
--eval-report leanctx=claude-code-leanctx.v1-leanctx.json
```
## What gets recorded
| Artifact | Contents |
|---|---|
| `runs/<id>/<instance>/<arm>/transcript.jsonl` | full agent stream (turns, tool calls, usage) |
| `…/model_patch.diff` | the submitted patch (`git add -A && git diff --cached`) |
| `…/meta.json` | exit code, wall time, billed tokens, cost, versions, flags |
| `runs/<id>/predictions-<arm>.jsonl` | official SWE-bench prediction format |
| `runs/<id>/result-v1.json` | canonical result artifact, self-hashing, embeds protocol + lock SHA-256 |
| `runs/<id>/result-v1.md` | human-readable endpoint table |
## Design notes
- **Fresh HOME per run** — the operator's machine has lean-ctx globally
installed; without isolation the native arm would be contaminated.
- **`--strict-mcp-config`** pins the MCP surface per arm explicitly: empty for
native, exactly the `lean-ctx init --agent claude` output for leanctx.
- **Usage comes from the runtime's own final report** (stream-json `result`
event) — no token estimation anywhere.
- **Selection is rule-based, not random** (sorted round-robin by repo), so the
subset is reproducible from the public dataset without a seed argument.
+18
View File
@@ -0,0 +1,18 @@
{
"n_tasks": 15,
"max_turns": 40,
"agent": {
"binary": "claude",
"output_format": "stream-json",
"extra_args": ["--verbose", "--dangerously-skip-permissions"]
},
"leanctx": {
"binary": "lean-ctx",
"init_args": ["init", "--agent", "claude"]
},
"dataset": "princeton-nlp/SWE-bench_Verified",
"split": "test",
"runs_dir": "runs",
"repo_cache_dir": ".cache/repos",
"timeout_seconds": 2400
}
+152
View File
@@ -0,0 +1,152 @@
# Faithful R2 lean-ctx arm
The R2 "Who Owns the Context Window?" round (Entelligentsia / tokbench, on the
Forge/forge-cli + pi runtime) is an oracle-free **planted-bug fix** task judged by
rebuild + reproduce. This directory holds the lean-ctx arm config so that
**"installed = running as designed"** — the R1 round ran lean-ctx with its
overhead defaults on, which cost a per-turn injected-prefix tax.
## What the faithful arm changes (vs R1 defaults)
| Lever | Setting | Why it matters on a phase-isolated harness |
|------|---------|--------------------------------------------|
| Zero injection | `rules_injection = off` | Drops the rule-file half of the ~3K-token per-turn prefix that R1 re-billed every turn. |
| Minimal surface | `minimal_overhead = true`, `tool_profile = minimal` | Drops the tool-schema half of that prefix (6-tool core, not the full surface). |
| Cold reads | `structure_first = true` | Biases `auto``map` for medium source files on a cold read (the only read saving that survives a fresh process), while capability guards keep suspect files full. |
| Shell routing | pi `mode = replace` / `routeShell = true` | Forces build/test/make output through `ctx_shell` (R1 saw 102 native bash / 0 ctx_shell — uncompressed). |
| Surface reach | `proxy_enabled = true`, `history_mode = cache-aware` | The proxy compresses the *whole* request body (incl. `forge_*` store output and native shell), with a byte-stable prefix so a cached rail keeps hitting. |
These are the three dominance vectors: **capability** (localize the defect in
fewer turns, never compress the suspect away), **surface** (proxy + shell), and
**honesty** (`lean-ctx gain` reports net-of-injection — see `meter-honest`).
## Files
- `lean-ctx.toml` — engine config. Copy to `$XDG_CONFIG_HOME/lean-ctx/config.toml`, or drop into the repo workspace as `.lean-ctx.toml`.
- `faithful-arm.env` — the same settings as env vars, plus the proxy base-URL wiring. Source it for harnesses that prefer env over files.
- `pi-config.json` — pi extension config (`~/.pi/agent/extensions/pi-lean-ctx/config.json`) for the pi/forge runtime.
## Run it (pi / forge-cli runtime — the R2 rail)
```bash
# 1. install config
mkdir -p ~/.pi/agent/extensions/pi-lean-ctx
cp bench/agent-task/r2/pi-config.json ~/.pi/agent/extensions/pi-lean-ctx/config.json
mkdir -p "${XDG_CONFIG_HOME:-$HOME/.config}/lean-ctx"
cp bench/agent-task/r2/lean-ctx.toml "${XDG_CONFIG_HOME:-$HOME/.config}/lean-ctx/config.toml"
# 2. start the wire-level proxy (foreground; background it for a run)
lean-ctx proxy start --port=4444 &
# 3. point the agent at the proxy
set -a; source bench/agent-task/r2/faithful-arm.env; set +a
```
## Run it (this repo's Claude harness)
`bench/agent-task` wires the `leanctx` arm purely via `lean-ctx init`
(`swebench_harness/run_arm.py`). To run it faithfully, apply the engine config
and proxy into the arm's fresh `HOME` and source `faithful-arm.env` before the
agent launches — the harness's own protocol stays unchanged.
## Verify the arm is actually faithful (preflight)
Run **before any priced run** — a green preflight is the gate devasur asked for:
it proves shell routes through `ctx_shell` (native `bash` suppressed) and is
actually compressed, not the R1 `102 native bash / 0 ctx_shell`:
```bash
node bench/agent-task/r2/preflight.mjs # resolves the installed pi config
node bench/agent-task/r2/preflight.mjs --config bench/agent-task/r2/pi-config.json
```
It checks the binary + version, that the config suppresses native `bash`
(`mode=replace` / `routeShell` — the unit-tested `resolveSuppressedBuiltins`
invariant), the embedded bridge, the faithful overhead levers, and measures real
shell compression. Exit code 1 if any hard gate fails.
Manual spot-checks (what the preflight automates):
```bash
lean-ctx config get rules_injection # -> off
lean-ctx config get tool_profile # -> minimal
lean-ctx proxy status # -> running on :4444, compression stats
lean-ctx gain # -> net_tokens_saved (net of injected overhead)
```
## tokbench PR offer
devasur invited patches on #361 ("we would appreciate your patch offer if you
can send us a PR"). The integration PR to tokbench is exactly this arm:
1. add the lean-ctx arm using `pi-config.json` + `lean-ctx.toml` above,
2. start `lean-ctx proxy start` for the arm and export `faithful-arm.env`,
3. the pi-extension `routeShell` / `mode=replace` fix already lives in
`packages/pi-lean-ctx` (it suppresses native `bash` so shell output reaches
the compressor without the agent having to choose `ctx_shell`); ship
`preflight.mjs` as the pre-run gate that proves it on the bench (the
green-preflight = "running as designed" devasur asked for).
Right-of-reply framing: lean-ctx is the only **code-aware** arm (localizes +
compresses without hiding the defect), the **broadest reach** via the proxy, and
the **only meter that reconciles to the provider bill**. rtk is shell-only and
architecturally capped; headroom is a blind wire compressor that under-compresses
code/prose by default and can compress bug-relevant content away.
## Version caveat + self-verify (ready to upstream to the tokbench README)
R1 measured lean-ctx with its **overhead defaults ON** (rules + full tool surface
injected every turn) on a phase-isolated harness — the single architecturesurface
combination where an injected per-turn prefix is pure re-billed cost, and *not*
the configuration lean-ctx recommends for that workload. We propose a short
version caveat at the top of the tokbench README plus a link to this faithful R2
arm:
> **Version caveat.** The lean-ctx figures below are from R1, run with the
> round's pinned lean-ctx version and default overhead injection **on**. On a
> phase-isolated harness that re-bills the per-turn prefix every turn. The
> maintainer-supplied **faithful R2 arm** (`rules_injection=off`,
> `tool_profile=minimal`, proxy + pi `mode=replace`) targets break-even there
> and a strict win on cache-priced / long-lived rails. See: `bench/agent-task/r2`.
### Self-verify (reproducible, local, no live model)
The advantage is provable **before** the priced R2 run — deterministically, with
no live model or secrets:
```bash
lean-ctx benchmark dual-arm # human table
lean-ctx benchmark dual-arm --json # machine-readable + BLAKE3 digest
```
Arm A = a stateless, phase-isolated session (the R1 shape); Arm B = the
long-lived proxy rail with cache-aware, byte-stable pruning. Costs come from real
token measurements (`o200k_base`) and published per-model pricing; the run is
fingerprinted by a BLAKE3 digest, so anyone can reproduce the exact figures.
Latest local self-verify — `medium` scenario, 15 turns, digest `f5ed145e61ce3689`,
150,574 raw input tok (phase-isolated) vs 4,164 lean prefix tok (long-lived):
| Model | Cache-priced? | Input-side saved |
|-------|---------------|------------------|
| claude-opus-4.5 | yes | 99.4 % |
| claude-sonnet-4.5 | yes | 99.4 % |
| gpt-5.4 | yes | 99.5 % |
| gemini-2.5-pro | no | 97.2 % |
| fallback-blended | no | 97.2 % |
Input-side, output held equal. Cache-priced rails show a strict win; non-caching
rails still net ahead on compression + the read-cache and are never worse than
break-even.
### Refreshed maintainer statement
> lean-ctx's R1 placement reflected its *default* overhead on a phase-isolated
> harness — the one architecturesurface combination where an injected per-turn
> prefix is pure re-billed cost. The faithful R2 arm removes that prefix
> (`rules_injection=off` + minimal surface) and routes the **whole** request body
> through the proxy with a byte-stable, cache-aware prefix. Our local dual-arm
> self-verify (deterministic, digest `f5ed145e61ce3689`) shows the result: a
> strict input-side win on cache-priced, long-lived rails and at least
> break-even everywhere else. lean-ctx remains the only code-aware arm and the
> only meter that reconciles to the provider bill (net-of-injection).
+26
View File
@@ -0,0 +1,26 @@
# Faithful R2 lean-ctx arm — env-var form (source before launching the agent).
#
# set -a; source bench/agent-task/r2/faithful-arm.env; set +a
#
# For harnesses that prefer env vars over config files. These mirror
# lean-ctx.toml exactly; an explicit env var always wins over the config file.
# Zero-injection prefix (neutralizes the R1 +38% tax).
export LEAN_CTX_RULES_INJECTION="off"
export LEAN_CTX_MINIMAL="1"
export LEAN_CTX_TOOL_PROFILE="minimal"
# Capability-safe-cheap cold reads (survives phase isolation).
export LEAN_CTX_STRUCTURE_FIRST="1"
# Cache-stable proxy history (keep prompt-cache hits on a cached rail).
export LEAN_CTX_PROXY_HISTORY_MODE="cache-aware"
# Proxy wiring. Start the foreground proxy first:
# lean-ctx proxy start --port=4444 &
# then point the agent at it. 127.0.0.1 is allow-listed, so no override flag is
# needed for the agent->proxy hop; the proxy->upstream hop uses api.anthropic.com.
export LEAN_CTX_PROXY_PORT="4444"
export ANTHROPIC_BASE_URL="http://127.0.0.1:4444"
export OPENAI_BASE_URL="http://127.0.0.1:4444/v1"
export GEMINI_API_BASE_URL="http://127.0.0.1:4444"
+43
View File
@@ -0,0 +1,43 @@
# Faithful R2 lean-ctx arm — engine config (config.toml).
#
# Copy to the lean-ctx config dir ($XDG_CONFIG_HOME/lean-ctx/config.toml) or drop
# into the repo workspace as `.lean-ctx.toml` (project-local override). Every key
# below is a real engine setting — no env var is required for these to take effect,
# though each also has a LEAN_CTX_* override (see faithful-arm.env).
#
# Goal: neutralize the R1 +38% injected-prefix tax and keep cold reads
# capability-safe-cheap on a phase-isolated harness.
# Zero-injection: do not inject rule files into the agent's context. The R1 tax
# was a ~3K-token per-turn prefix (rule file + tool schemas) re-billed every turn
# on a non-caching rail. "off" removes the rule-file half entirely.
rules_injection = "off"
# Minimal overhead: trims the per-tool schema/preamble surface so the fixed
# injected cost per turn is as small as possible.
minimal_overhead = true
# Minimal tool profile: expose the 6-tool core instead of the full ~15 ctx_*
# surface, shrinking the other half of the injected prefix (the tool schemas).
tool_profile = "minimal"
# Structure-first cold reads: on a fresh process (no warm session cache to
# amortize a full read) bias `auto` toward `map` for medium source files. This
# is the only read saving that survives phase isolation, and it aids
# localization (deps + located API + task-relevant body). All capability guards
# stay in force: active_diagnostic / edit-fail / small-file / task-named files
# are always read full.
structure_first = true
# Proxy: the cross-tool wire compressor. The only path that reaches another
# tool's output (e.g. forge_* store reads) and native shell/build/test logs, so
# "installed = compressing the whole addressable surface", not just ctx_* tools.
proxy_enabled = true
proxy_port = 4444
[proxy]
# cache-aware history pruning keeps the request prefix byte-stable so a
# cache-priced rail (Anthropic) keeps hitting the prompt cache (#498). Never use
# "rolling" on a cached rail — it rewrites a stable message every turn and turns
# cheap cache reads into full-price writes.
history_mode = "cache-aware"
+12
View File
@@ -0,0 +1,12 @@
{
"mode": "replace",
"routeShell": true,
"enableMcp": true,
"env": {
"LEAN_CTX_RULES_INJECTION": "off",
"LEAN_CTX_MINIMAL": "1",
"LEAN_CTX_TOOL_PROFILE": "minimal",
"LEAN_CTX_STRUCTURE_FIRST": "1",
"LEAN_CTX_PROXY_HISTORY_MODE": "cache-aware"
}
}
+232
View File
@@ -0,0 +1,232 @@
#!/usr/bin/env node
// R2 faithful-arm preflight — prove "installed = running as designed" before a
// priced run. Targets the R1 finding that the agent logged 102 native `bash`
// calls and 0 `ctx_shell` calls, so the heaviest addressable surface in a fix
// task (make / reproducer / test logs) never reached the compressor (#361).
//
// It verifies, on THIS machine, that:
// 1. the lean-ctx binary is present (and reports its version),
// 2. the resolved pi config suppresses native `bash` (mode=replace or
// routeShell), so the agent must use `ctx_shell` — the suppression itself
// is the unit-tested invariant `resolveSuppressedBuiltins` in
// packages/pi-lean-ctx/extensions/config.ts,
// 3. the embedded MCP bridge / session cache is enabled,
// 4. the faithful-arm overhead levers are set (rules_injection=off,
// tool_profile=minimal, structure_first),
// 5. lean-ctx actually compresses shell output (measured: smaller than raw).
//
// A green run is the precondition devasur asked for: shell routes through
// ctx_shell and is metered, not native bash.
//
// Usage:
// node bench/agent-task/r2/preflight.mjs [--config <path>]
//
// POSIX shell is assumed for the compression probe (the R2 rail is the
// forge-cli / pi runtime on Linux). Exit code 1 if any hard gate fails.
import { execFileSync } from "node:child_process";
import { existsSync, readFileSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
const RECOMMENDED_MIN_VERSION = "3.8.6"; // anti-inflation guard + bridged ctx_read
// ── tiny PASS/FAIL/WARN reporter ──────────────────────────────────────────
let failed = false;
const pass = (name, detail) => console.log(` [PASS] ${name.padEnd(24)} ${detail}`);
const warn = (name, detail) => console.log(` [WARN] ${name.padEnd(24)} ${detail}`);
const fail = (name, detail) => {
console.log(` [FAIL] ${name.padEnd(24)} ${detail}`);
failed = true;
};
const gate = (ok, name, okDetail, failDetail) =>
ok ? pass(name, okDetail) : fail(name, failDetail);
// ── config resolution (mirrors extensions/config.ts precedence) ───────────
function envFlag(name) {
const raw = process.env[name];
if (!raw) return false;
const v = raw.trim().toLowerCase();
return v === "1" || v === "true" || v === "yes" || v === "on";
}
function parseConfigArg() {
const i = process.argv.indexOf("--config");
return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : undefined;
}
function resolveConfigPath() {
const explicit = parseConfigArg();
if (explicit) return explicit;
const installed = resolve(
homedir(), ".pi", "agent", "extensions", "pi-lean-ctx", "config.json",
);
if (existsSync(installed)) return installed;
// Fall back to this repo's reference config so the preflight self-tests
// locally even when the extension is not installed on the dev machine.
return resolve(SCRIPT_DIR, "pi-config.json");
}
function readConfig(path) {
if (!existsSync(path)) return {};
try {
const parsed = JSON.parse(readFileSync(path, "utf8"));
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
} catch (err) {
fail("config.json", `invalid JSON at ${path} (${err.message})`);
return {};
}
}
// Effective env value: an explicit process env always wins over the config
// `env` block the extension forwards to every lean-ctx subprocess.
function effectiveEnv(cfg, key) {
return process.env[key] ?? (cfg.env && typeof cfg.env === "object" ? cfg.env[key] : undefined);
}
function resolveMode(cfg) {
const raw = (process.env.LEAN_CTX_PI_MODE ?? cfg.mode ?? "additive").toLowerCase();
return raw === "replace" ? "replace" : "additive";
}
function resolveRouteShell(mode, cfg) {
if (mode === "replace") return true;
if (process.env.LEAN_CTX_PI_ROUTE_SHELL !== undefined) return envFlag("LEAN_CTX_PI_ROUTE_SHELL");
return cfg.routeShell === true;
}
function resolveEnableMcp(cfg) {
return process.env.LEAN_CTX_PI_ENABLE_MCP !== undefined
? envFlag("LEAN_CTX_PI_ENABLE_MCP")
: cfg.enableMcp !== false;
}
function resolveBinary() {
return process.env.LEAN_CTX_BIN || "lean-ctx";
}
function compareSemver(a, b) {
const pa = a.split(".").map(Number);
const pb = b.split(".").map(Number);
for (let i = 0; i < 3; i++) {
const d = (pa[i] || 0) - (pb[i] || 0);
if (d !== 0) return d < 0 ? -1 : 1;
}
return 0;
}
// ── individual checks ─────────────────────────────────────────────────────
function checkBinary(bin) {
try {
const out = execFileSync(bin, ["--version"], { encoding: "utf8" }).trim();
pass("lean-ctx binary", out);
const m = out.match(/(\d+\.\d+\.\d+)/);
if (m && compareSemver(m[1], RECOMMENDED_MIN_VERSION) < 0) {
warn("version", `${m[1]} < ${RECOMMENDED_MIN_VERSION} — pin a release with the anti-inflation + routeShell fixes`);
}
return true;
} catch (err) {
fail("lean-ctx binary", `not runnable (${err.message}) — set LEAN_CTX_BIN or add lean-ctx to PATH`);
return false;
}
}
function checkShellRouting(mode, routeShell) {
// resolveSuppressedBuiltins (config.ts, unit-tested): replace ⇒ all natives,
// additive+routeShell ⇒ just bash, additive ⇒ none. bash gone ⟺ the agent
// cannot pick native bash and must use ctx_shell.
const bashSuppressed = mode === "replace" || routeShell;
gate(
bashSuppressed,
"shell routing",
`native bash suppressed (mode=${mode}, routeShell=${routeShell}) → ctx_shell only`,
`native bash still exposed (mode=${mode}, routeShell=${routeShell}) — set "mode":"replace" or "routeShell":true, else the agent reproduces R1's 102 bash / 0 ctx_shell`,
);
}
function checkBridge(enableMcp) {
gate(
enableMcp,
"session cache",
"embedded MCP bridge enabled (unchanged re-reads ~13 tokens)",
'embedded bridge disabled — remove "enableMcp":false / LEAN_CTX_PI_ENABLE_MCP=0',
);
}
function checkFaithfulLevers(cfg) {
const rules = (effectiveEnv(cfg, "LEAN_CTX_RULES_INJECTION") || "").toLowerCase();
gate(rules === "off", "rules_injection", "off (no per-turn rule-file prefix)", `"${rules || "unset"}" — set LEAN_CTX_RULES_INJECTION=off`);
const profile = (effectiveEnv(cfg, "LEAN_CTX_TOOL_PROFILE") || "").toLowerCase();
gate(profile === "minimal", "tool_profile", "minimal (6-tool core)", `"${profile || "unset"}" — set LEAN_CTX_TOOL_PROFILE=minimal`);
const structureRaw = effectiveEnv(cfg, "LEAN_CTX_STRUCTURE_FIRST");
const structureOn = ["1", "true", "yes", "on"].includes((structureRaw || "").toLowerCase());
gate(structureOn, "structure_first", "on (capability-safe cold-read bias)", `"${structureRaw || "unset"}" — set LEAN_CTX_STRUCTURE_FIRST=1`);
}
function checkCompression(bin) {
// Real proof that shell output is compressed (and therefore metered), without
// depending on a footer string: run a log-like command raw vs through
// lean-ctx and assert the lean-ctx output is strictly smaller. The generator is
// a single `awk` BEGIN loop: it avoids shell command-substitution (so the probe
// runs under shell_strict_mode) and `awk` is in the default shell_allowlist —
// unlike `seq`, which mode=replace blocks, making the probe false-fail (#361).
const cmd =
'awk \'BEGIN { for (i = 1; i <= 80; i++) printf "[INFO] building module %d of 80 ... ok\\n", i }\'';
try {
const raw = execFileSync("/bin/sh", ["-c", cmd], { encoding: "utf8" });
const compressed = execFileSync(bin, ["-c", cmd], {
encoding: "utf8",
env: { ...process.env, LEAN_CTX_COMPRESS: "1", LEAN_CTX_SAVINGS_FOOTER: "always" },
});
const pct = raw.length > 0 ? Math.round((1 - compressed.length / raw.length) * 100) : 0;
gate(
compressed.length < raw.length,
"shell compression",
`${raw.length}${compressed.length} bytes (-${pct}%) via ctx_shell path`,
`lean-ctx output (${compressed.length}B) not smaller than raw (${raw.length}B)`,
);
} catch (err) {
fail("shell compression", `probe failed (${err.message})`);
}
}
function checkProxy(bin) {
// Soft: the proxy is usually started just-in-time for the run. Report status
// when reachable so a stale/missing proxy is visible, but never gate on it.
try {
const out = execFileSync(bin, ["proxy", "status"], { encoding: "utf8" }).trim();
const running = /running|listening|:\d+/i.test(out) && !/not running|stopped/i.test(out);
(running ? pass : warn)("proxy", out.split("\n")[0] || "status reported");
} catch {
warn("proxy", "not running — start with `lean-ctx proxy start --port=4444` before the run");
}
}
// ── main ──────────────────────────────────────────────────────────────────
const configPath = resolveConfigPath();
const cfg = readConfig(configPath);
const mode = resolveMode(cfg);
const routeShell = resolveRouteShell(mode, cfg);
const bin = resolveBinary();
console.log("R2 faithful-arm preflight");
console.log(` config: ${configPath}${existsSync(configPath) ? "" : " (absent — using env + defaults)"}`);
console.log("");
checkBinary(bin);
checkShellRouting(mode, routeShell);
checkBridge(resolveEnableMcp(cfg));
checkFaithfulLevers(cfg);
checkCompression(bin);
checkProxy(bin);
console.log("");
if (failed) {
console.log("preflight FAILED — fix the gates above before any priced run.");
process.exit(1);
}
console.log("preflight PASSED — installed = running as designed.");
+5
View File
@@ -0,0 +1,5 @@
# Agent-Task-Benchmark v1 (GL #493) — Python >= 3.9
# Selection (one-time): pulls the public SWE-bench Verified dataset.
datasets>=2.19
# Official evaluation harness (docker required at evaluation time).
swebench>=2.1
@@ -0,0 +1,52 @@
"""Agent-Task-Benchmark v1 harness (GL #493) — shared helpers.
Measures task success rate and cost per solved task for an agentic coding
workload (SWE-bench Verified subset), with and without lean-ctx, under the
pre-registered protocol in ../PROTOCOL.md.
"""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
BENCH_ROOT = Path(__file__).resolve().parent.parent
ARMS = ("native", "leanctx")
def load_config() -> dict:
return json.loads((BENCH_ROOT / "config.json").read_text())
def load_tasks_lock() -> list:
lock = BENCH_ROOT / "tasks.lock.json"
if not lock.exists():
raise SystemExit(
"tasks.lock.json missing — generate it once with: python -m swebench_harness.select_tasks"
)
return json.loads(lock.read_text())["instances"]
def canonical_dumps(obj) -> str:
"""Stable JSON for hashing: sorted keys, no float surprises, no whitespace drift."""
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
def sha256_text(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def sha256_file(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def read_jsonl(path: Path) -> list:
rows = []
with path.open() as fh:
for line in fh:
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
@@ -0,0 +1,55 @@
"""Assemble per-arm predictions.jsonl for the official SWE-bench evaluation.
Usage:
python -m swebench_harness.collect --run-id v1
Then evaluate each arm (docker required):
python -m swebench.harness.run_evaluation \
--dataset_name princeton-nlp/SWE-bench_Verified \
--predictions_path runs/v1/predictions-native.jsonl \
--run_id v1-native --max_workers 4
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from . import ARMS, BENCH_ROOT, load_config, load_tasks_lock
def collect_arm(run_root: Path, arm: str, instances: list) -> Path:
out_path = run_root / f"predictions-{arm}.jsonl"
rows, missing = [], []
for inst in instances:
iid = inst["instance_id"]
patch_file = run_root / iid / arm / "model_patch.diff"
if not patch_file.exists():
missing.append(iid)
continue
rows.append({
"instance_id": iid,
"model_name_or_path": f"claude-code-{arm}",
"model_patch": patch_file.read_text(),
})
with out_path.open("w") as fh:
for row in rows:
fh.write(json.dumps(row) + "\n")
print(f"{arm}: {len(rows)} predictions -> {out_path}" + (f" (missing: {', '.join(missing)})" if missing else ""))
return out_path
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--run-id", required=True)
args = ap.parse_args()
cfg = load_config()
run_root = BENCH_ROOT / cfg["runs_dir"] / args.run_id
instances = load_tasks_lock()
for arm in ARMS:
collect_arm(run_root, arm, instances)
if __name__ == "__main__":
main()
@@ -0,0 +1,99 @@
"""Preflight: verify the machine can run the benchmark before spending money.
Checks everything the runbook needs — binaries, auth, docker, network, the
frozen task lock — and prints one PASS/FAIL line each. Exit 1 if anything
required for the requested stage is missing.
Usage:
python3 -m swebench_harness.preflight # checks for agent runs
python3 -m swebench_harness.preflight --evaluate # also checks docker
"""
from __future__ import annotations
import argparse
import json
import os
import shutil
import subprocess
import sys
import urllib.request
from . import BENCH_ROOT, load_config
def run(cmd: list) -> "tuple[int, str]":
try:
res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=30)
return res.returncode, (res.stdout or "").strip()
except FileNotFoundError:
return 127, "not found"
except subprocess.TimeoutExpired:
return 124, "timed out"
class Preflight:
def __init__(self) -> None:
self.failed = False
def check(self, name: str, ok: bool, detail: str) -> None:
print(f" [{'PASS' if ok else 'FAIL'}] {name:<28} {detail}")
if not ok:
self.failed = True
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--evaluate", action="store_true", help="also check the docker evaluation prerequisites")
args = ap.parse_args()
cfg = load_config()
pf = Preflight()
print("agent-run prerequisites:")
code, out = run([cfg["agent"]["binary"], "--version"])
pf.check("claude CLI", code == 0, out.splitlines()[0] if out else "not found")
code, out = run([cfg["leanctx"]["binary"], "--version"])
pf.check("lean-ctx CLI", code == 0, out.splitlines()[0] if out else "not found")
code, out = run(["git", "--version"])
pf.check("git", code == 0, out)
has_key = bool(os.environ.get("ANTHROPIC_API_KEY") or os.environ.get("ANTHROPIC_AUTH_TOKEN"))
pf.check(
"ANTHROPIC_API_KEY",
has_key,
"set" if has_key else "missing — fresh-HOME runs have no stored login (export it first)",
)
lock = BENCH_ROOT / "tasks.lock.json"
if lock.exists():
n = json.loads(lock.read_text())["n"]
pf.check("tasks.lock.json", True, f"frozen, n={n}")
else:
pf.check("tasks.lock.json", False, "missing — python3 -m swebench_harness.select_tasks (one-time)")
try:
with urllib.request.urlopen("https://github.com", timeout=10) as resp:
pf.check("github.com reachable", resp.status < 500, f"HTTP {resp.status} (repo mirrors clone from here)")
except OSError as e:
pf.check("github.com reachable", False, str(e))
free_gb = shutil.disk_usage(BENCH_ROOT).free / 1e9
pf.check("disk space", free_gb > 20, f"{free_gb:.0f} GB free (mirrors ~2 GB, eval images need more)")
if args.evaluate:
print("evaluation prerequisites:")
code, out = run(["docker", "info", "--format", "{{.ServerVersion}}"])
pf.check("docker daemon", code == 0, f"server {out}" if code == 0 else out[-120:])
code, out = run([sys.executable, "-c", "import swebench; print(swebench.__version__)"])
pf.check("swebench package", code == 0, out if code == 0 else "pip install -r requirements.txt")
if pf.failed:
sys.exit(1)
print("all checks passed.")
if __name__ == "__main__":
main()
+126
View File
@@ -0,0 +1,126 @@
"""Compute endpoints (PROTOCOL.md §5) and emit the verifiable result artifact.
Inputs: per-run meta.json files + the official SWE-bench evaluation reports.
Output: runs/<id>/result-v1.json (canonical, self-hashing) + markdown summary.
Usage:
python -m swebench_harness.report --run-id v1 \
--eval-report native=claude-code-native.v1-native.json \
--eval-report leanctx=claude-code-leanctx.v1-leanctx.json
"""
from __future__ import annotations
import argparse
import json
import statistics
import sys
from pathlib import Path
from . import ARMS, BENCH_ROOT, canonical_dumps, load_config, load_tasks_lock, sha256_file, sha256_text
def arm_metrics(run_root: Path, arm: str, instances: list, resolved_ids: set) -> dict:
metas = []
for inst in instances:
meta_file = run_root / inst["instance_id"] / arm / "meta.json"
if meta_file.exists():
metas.append(json.loads(meta_file.read_text()))
n = len(metas)
resolved = sum(1 for m in metas if m["instance_id"] in resolved_ids)
usable = [m for m in metas if not m.get("usage_missing")]
total_cost = round(sum(m.get("total_cost_usd") or 0.0 for m in usable), 4)
input_tokens = [m["input_tokens"] for m in usable if m.get("input_tokens") is not None]
output_tokens = [m["output_tokens"] for m in usable if m.get("output_tokens") is not None]
return {
"n_run": n,
"resolved": resolved,
"resolved_rate": round(resolved / n, 4) if n else None,
"usage_missing_runs": n - len(usable),
"total_cost_usd": total_cost,
"cost_per_resolved_usd": round(total_cost / resolved, 4) if resolved else None,
"median_input_tokens": statistics.median(input_tokens) if input_tokens else None,
"median_output_tokens": statistics.median(output_tokens) if output_tokens else None,
"median_wall_time_seconds": statistics.median(m["wall_time_seconds"] for m in metas) if metas else None,
"timed_out_runs": sum(1 for m in metas if m.get("timed_out")),
"resolved_instance_ids": sorted(m["instance_id"] for m in metas if m["instance_id"] in resolved_ids),
}
def load_resolved_ids(report_path: Path) -> set:
report = json.loads(report_path.read_text())
ids = report.get("resolved_ids")
if ids is None:
sys.exit(f"{report_path}: no resolved_ids field — pass the official run_evaluation report")
return set(ids)
def render_markdown(result: dict) -> str:
lines = [
"# Agent-Task-Benchmark v1 — result",
"",
f"run_id `{result['run_id']}` · N={result['n_tasks']} (SWE-bench Verified subset) · protocol sha256 `{result['protocol_sha256'][:16]}…`",
"",
"| endpoint | native | leanctx |",
"|---|---|---|",
]
rows = [
("resolved", "resolved"),
("resolved rate", "resolved_rate"),
("total cost (USD)", "total_cost_usd"),
("cost / resolved task (USD)", "cost_per_resolved_usd"),
("median billed input tokens", "median_input_tokens"),
("median output tokens", "median_output_tokens"),
("median wall time (s)", "median_wall_time_seconds"),
("timed-out runs", "timed_out_runs"),
]
for label, key in rows:
lines.append(f"| {label} | {result['arms']['native'][key]} | {result['arms']['leanctx'][key]} |")
lines += ["", f"artifact sha256: `{result['artifact_sha256']}`", ""]
return "\n".join(lines)
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--run-id", required=True)
ap.add_argument("--eval-report", action="append", required=True,
metavar="ARM=PATH", help="official evaluation report per arm")
args = ap.parse_args()
reports = {}
for spec in args.eval_report:
arm, _, path = spec.partition("=")
if arm not in ARMS or not path:
sys.exit(f"bad --eval-report '{spec}' (expected ARM=PATH)")
reports[arm] = Path(path)
if set(reports) != set(ARMS):
sys.exit(f"need eval reports for both arms: {ARMS}")
cfg = load_config()
instances = load_tasks_lock()
run_root = BENCH_ROOT / cfg["runs_dir"] / args.run_id
result = {
"benchmark": "lean-ctx agent-task v1 (GL #493)",
"run_id": args.run_id,
"n_tasks": len(instances),
"protocol_sha256": sha256_file(BENCH_ROOT / "PROTOCOL.md"),
"tasks_lock_sha256": sha256_file(BENCH_ROOT / "tasks.lock.json"),
"arms": {
arm: arm_metrics(run_root, arm, instances, load_resolved_ids(reports[arm]))
for arm in ARMS
},
}
result["artifact_sha256"] = sha256_text(canonical_dumps(result))
out_json = run_root / "result-v1.json"
out_json.write_text(canonical_dumps(result) + "\n")
out_md = run_root / "result-v1.md"
out_md.write_text(render_markdown(result))
print(render_markdown(result))
print(f"wrote {out_json} + {out_md}")
print("sign it: ssh-keygen -Y sign -f ~/.ssh/id_ed25519 -n lean-ctx-bench " + str(out_json))
if __name__ == "__main__":
main()
@@ -0,0 +1,236 @@
"""Run one benchmark arm over the locked task set (PROTOCOL.md §3-§4).
Per (instance, arm): clean checkout of base_commit → identical prompt →
Claude Code headless → transcript.jsonl + model_patch.diff + meta.json.
Isolation: every run gets a fresh HOME so the operator's global lean-ctx /
agent config cannot bleed into either arm. The `leanctx` arm gets its MCP
wiring exclusively from `lean-ctx init --agent claude` inside the workspace.
Usage:
python -m swebench_harness.run_arm --arm native --run-id v1
python -m swebench_harness.run_arm --arm leanctx --run-id v1 [--instance ID]
"""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from . import ARMS, BENCH_ROOT, load_config, load_tasks_lock
def sh(cmd: list, cwd: Path = None, env: dict = None, timeout: int = None) -> "subprocess.CompletedProcess":
return subprocess.run(
cmd, cwd=str(cwd) if cwd else None, env=env, timeout=timeout,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
)
def ensure_repo_mirror(repo: str, cache_dir: Path) -> Path:
mirror = cache_dir / f"{repo.replace('/', '__')}.git"
if not mirror.exists():
mirror.parent.mkdir(parents=True, exist_ok=True)
print(f" mirror-clone {repo}")
res = sh(["git", "clone", "--mirror", f"https://github.com/{repo}.git", str(mirror)])
if res.returncode != 0:
raise RuntimeError(f"mirror clone failed for {repo}:\n{res.stdout[-2000:]}")
return mirror
def checkout_workspace(mirror: Path, base_commit: str, workspace: Path) -> None:
res = sh(["git", "clone", "--no-checkout", str(mirror), str(workspace / "repo")])
if res.returncode != 0:
raise RuntimeError(f"clone from mirror failed:\n{res.stdout[-2000:]}")
res = sh(["git", "checkout", "--force", base_commit], cwd=workspace / "repo")
if res.returncode != 0:
raise RuntimeError(f"checkout {base_commit} failed:\n{res.stdout[-2000:]}")
def fresh_home(run_dir: Path) -> dict:
home = run_dir / "home"
home.mkdir(parents=True, exist_ok=True)
env = {
"HOME": str(home),
"PATH": os.environ["PATH"],
"GIT_TERMINAL_PROMPT": "0",
"TERM": "dumb",
}
for key in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_BASE_URL"):
if os.environ.get(key):
env[key] = os.environ[key]
if "ANTHROPIC_API_KEY" not in env and "ANTHROPIC_AUTH_TOKEN" not in env:
sys.exit("ANTHROPIC_API_KEY (or ANTHROPIC_AUTH_TOKEN) must be set — fresh-HOME runs have no stored login.")
return env
def setup_leanctx(cfg: dict, repo_dir: Path, env: dict, run_dir: Path) -> Path:
"""`lean-ctx init --agent claude` in the workspace, then pin the MCP surface.
init registers the MCP server in the fresh HOME's `~/.claude.json` (plus
the CLAUDE.md rules in the repo). Relying on that implicit user-scope
lookup would make the arm fragile, so the `mcpServers` block is extracted
into an explicit config the agent is pinned to via `--strict-mcp-config`.
A missing registration is a hard error — the arm must never silently run
without lean-ctx.
"""
binary = cfg["leanctx"]["binary"]
res = sh([binary, *cfg["leanctx"]["init_args"]], cwd=repo_dir, env=env)
(run_dir / "leanctx-init.log").write_text(res.stdout)
if res.returncode != 0:
raise RuntimeError(f"lean-ctx init failed (see {run_dir / 'leanctx-init.log'})")
servers = {}
for candidate in (repo_dir / ".mcp.json", Path(env["HOME"]) / ".claude.json"):
if candidate.exists():
servers = json.loads(candidate.read_text()).get("mcpServers") or {}
if "lean-ctx" in servers:
break
if "lean-ctx" not in servers:
raise RuntimeError("lean-ctx init left no MCP registration — leanctx arm would be inert")
mcp_config = run_dir / "mcp-config.json"
mcp_config.write_text(json.dumps({"mcpServers": {"lean-ctx": servers["lean-ctx"]}}, indent=2) + "\n")
return mcp_config
def agent_cmd(cfg: dict, prompt: str, mcp_config: "Path | None") -> list:
cmd = [
cfg["agent"]["binary"], "-p", prompt,
"--output-format", cfg["agent"]["output_format"],
"--max-turns", str(cfg["max_turns"]),
*cfg["agent"]["extra_args"],
]
# Both arms get a hard-pinned MCP surface: empty for native, exactly the
# lean-ctx registration for leanctx (PROTOCOL.md §3).
if mcp_config is None:
cmd += ["--strict-mcp-config", "--mcp-config", '{"mcpServers":{}}']
else:
cmd += ["--strict-mcp-config", "--mcp-config", str(mcp_config)]
return cmd
def parse_usage(transcript: Path) -> dict:
"""Extract the runtime's own final usage report (stream-json `result` event)."""
result = {}
try:
with transcript.open() as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
event = json.loads(line)
except json.JSONDecodeError:
continue
if event.get("type") == "result":
result = event
except OSError:
pass
usage = result.get("usage") or {}
return {
"usage_missing": not usage,
"input_tokens": usage.get("input_tokens"),
"output_tokens": usage.get("output_tokens"),
"cache_creation_input_tokens": usage.get("cache_creation_input_tokens"),
"cache_read_input_tokens": usage.get("cache_read_input_tokens"),
"total_cost_usd": result.get("total_cost_usd"),
"num_turns": result.get("num_turns"),
"is_error": result.get("is_error"),
"subtype": result.get("subtype"),
}
def extract_patch(repo_dir: Path, out: Path) -> bool:
sh(["git", "add", "-A"], cwd=repo_dir)
res = sh(["git", "diff", "--cached"], cwd=repo_dir)
out.write_text(res.stdout)
return bool(res.stdout.strip())
def tool_versions(cfg: dict, arm: str, env: dict) -> dict:
versions = {}
res = sh([cfg["agent"]["binary"], "--version"], env=env)
versions["agent"] = res.stdout.strip().splitlines()[0] if res.stdout else "unknown"
if arm == "leanctx":
res = sh([cfg["leanctx"]["binary"], "--version"], env=env)
versions["leanctx"] = res.stdout.strip().splitlines()[0] if res.stdout else "unknown"
return versions
def run_instance(cfg: dict, inst: dict, arm: str, run_root: Path, prompt_template: str) -> dict:
iid = inst["instance_id"]
run_dir = run_root / iid / arm
if (run_dir / "meta.json").exists():
print(f" {iid}/{arm}: already done, skipping")
return json.loads((run_dir / "meta.json").read_text())
run_dir.mkdir(parents=True, exist_ok=True)
mirror = ensure_repo_mirror(inst["repo"], BENCH_ROOT / cfg["repo_cache_dir"])
checkout_workspace(mirror, inst["base_commit"], run_dir)
repo_dir = run_dir / "repo"
env = fresh_home(run_dir)
mcp_config = setup_leanctx(cfg, repo_dir, env, run_dir) if arm == "leanctx" else None
prompt = prompt_template.replace("{problem_statement}", inst["problem_statement"])
cmd = agent_cmd(cfg, prompt, mcp_config)
print(f" {iid}/{arm}: running agent …")
started = time.time()
timed_out = False
with (run_dir / "transcript.jsonl").open("w") as out:
try:
proc = subprocess.run(
cmd, cwd=str(repo_dir), env=env, stdout=out,
stderr=subprocess.STDOUT, timeout=cfg["timeout_seconds"],
)
agent_exit = proc.returncode
except subprocess.TimeoutExpired:
timed_out = True
agent_exit = -1
wall = round(time.time() - started, 1)
meta = {
"instance_id": iid,
"arm": arm,
"agent_exit_code": agent_exit,
"timed_out": timed_out,
"wall_time_seconds": wall,
"has_patch": extract_patch(repo_dir, run_dir / "model_patch.diff"),
"versions": tool_versions(cfg, arm, env),
**parse_usage(run_dir / "transcript.jsonl"),
}
(run_dir / "meta.json").write_text(json.dumps(meta, indent=2) + "\n")
print(f" {iid}/{arm}: exit={agent_exit} wall={wall}s patch={meta['has_patch']} cost={meta['total_cost_usd']}")
return meta
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--arm", choices=ARMS, required=True)
ap.add_argument("--run-id", required=True)
ap.add_argument("--instance", help="run a single instance_id (smoke test)")
args = ap.parse_args()
cfg = load_config()
instances = load_tasks_lock()
if args.instance:
instances = [i for i in instances if i["instance_id"] == args.instance]
if not instances:
sys.exit(f"instance {args.instance} not in tasks.lock.json")
prompt_template = (BENCH_ROOT / "PROMPT.md").read_text()
run_root = BENCH_ROOT / cfg["runs_dir"] / args.run_id
print(f"arm={args.arm} run_id={args.run_id} instances={len(instances)}")
for inst in instances:
run_instance(cfg, inst, args.arm, run_root, prompt_template)
if __name__ == "__main__":
main()
@@ -0,0 +1,76 @@
"""Deterministic task selection → tasks.lock.json (run exactly once; see PROTOCOL.md §2).
Selection rule (pre-registered): sort all SWE-bench-Verified instances by
instance_id; group by repo; visit repos in ascending name order round-robin,
taking the lexicographically first untaken instance per repo each cycle,
until N are selected. No seed, no randomness — fully reproducible from the
public dataset.
The lock embeds everything the runner needs (problem statement, base commit),
so benchmark runs do not depend on Hugging Face availability.
"""
from __future__ import annotations
import json
import sys
from collections import OrderedDict
from . import BENCH_ROOT, canonical_dumps, load_config, sha256_text
def select(instances: list, n: int) -> list:
by_repo: "OrderedDict[str, list]" = OrderedDict()
for inst in sorted(instances, key=lambda r: r["instance_id"]):
by_repo.setdefault(inst["repo"], []).append(inst)
picked = []
while len(picked) < n:
progressed = False
for repo in sorted(by_repo):
if by_repo[repo]:
picked.append(by_repo[repo].pop(0))
progressed = True
if len(picked) == n:
break
if not progressed:
break
return picked
def main() -> None:
lock_path = BENCH_ROOT / "tasks.lock.json"
if lock_path.exists():
sys.exit(f"{lock_path} already exists — the v1 lock is frozen (PROTOCOL.md §2).")
cfg = load_config()
from datasets import load_dataset # heavy import, only needed here
ds = load_dataset(cfg["dataset"], split=cfg["split"])
picked = select(list(ds), cfg["n_tasks"])
keep = [
"instance_id",
"repo",
"base_commit",
"environment_setup_commit",
"version",
"problem_statement",
]
instances = [{k: inst[k] for k in keep} for inst in picked]
payload = {
"dataset": cfg["dataset"],
"split": cfg["split"],
"selection_rule": "sorted-round-robin-by-repo (PROTOCOL.md §2)",
"n": len(instances),
"instances": instances,
}
text = canonical_dumps(payload)
lock_path.write_text(text + "\n")
print(f"wrote {lock_path} ({len(instances)} instances, sha256 {sha256_text(text)[:16]}…)")
for inst in instances:
print(f" {inst['instance_id']}")
if __name__ == "__main__":
main()
File diff suppressed because one or more lines are too long
+109
View File
@@ -0,0 +1,109 @@
# compress() Benchmark — lean-ctx vs Headroom
A head-to-head compression benchmark for the drop-in `compress(messages, model)`
contract. It runs **both** libraries over the *same* real corpus with the *same*
tokenizer and emits JSON (compression ratio + latency).
Numbers are always **measured, never fabricated**: a tool that is not installed
or whose daemon is unreachable is reported as `available: false` (with an install
hint) instead of being estimated.
## Prerequisites
| For | Install |
|-----|---------|
| lean-ctx side | a daemon serving `POST /v1/compress``lean-ctx dev-install` |
| Headroom side (optional) | `pip install headroom-ai` |
| Accurate token counts (optional) | `pip install tiktoken` (else character counts) |
The lean-ctx Python SDK is imported directly from `packages/python-lean-ctx`
(no `pip install` required).
## Run
```bash
# Default corpus = docs/reference/*.md, model = gpt-4o
python bench/compress/benchmark.py
# Custom corpus, write the JSON log to a file
python bench/compress/benchmark.py --corpus docs/ --model gpt-4o --out report.json
# Bound the corpus size
python bench/compress/benchmark.py --max-files 50 --max-bytes 200000
```
The corpus is built from **real on-disk files** (`.md .rs .py .ts .txt .json .log
.yaml .yml`) under `--corpus`; one `user` message per file. No fixtures, no mock
payloads.
## Output
```json
{
"corpus": { "path": "docs/reference", "messages": 27, "model": "gpt-4o", "tokenizer": "o200k_base" },
"lean_ctx": { "available": true, "original_tokens": 69594, "compressed_tokens": 57615,
"tokens_saved": 11979, "ratio": 0.172, "latency_ms": 1014.03 },
"headroom": { "available": false, "install": "pip install headroom-ai" }
}
```
`ratio = 1 compressed/original`, computed with the shared tokenizer for both
tools so the comparison is apples-to-apples.
## Daemon-free lean-ctx benchmark
To benchmark the deterministic funnel without a running daemon (calls the Rust
`compress_messages` directly, `o200k_base`):
```bash
cargo test -p lean-ctx --lib \
proxy::compress_api::tests::bench_real_corpus_o200k -- --ignored --nocapture
```
## Extractive vs truncation (prose quality)
Prose is the one place a rule-based funnel can only truncate. This benchmark runs
the **extractive** ranker (centrality, reusing the shipped all-MiniLM model)
head-to-head against FIFO **truncation** at an identical 50% char budget over the
real `docs/reference` corpus, and reports both token savings AND a coverage
quality per method:
```bash
cargo test -p lean-ctx --lib --features embeddings \
core::extractive::tests::bench_extractive_vs_truncation -- --ignored --nocapture
```
The report has two honest signals:
- **`avg_coverage`** (query-free): mean, over every full-document sentence, of its
cosine to the **nearest kept sentence**`1.0` means the kept set has a close
match for every original sentence. Coverage is the fair extractive-quality proxy:
a centroid cosine would *reward redundancy* (a contiguous prefix tracks the
whole-doc centroid while MMR de-duplication deliberately diversifies away from
it), whereas coverage rewards spreading the budget across the whole document.
- **`rag_query_recall`** (query-aware): a real sentence from each document's *back
half* is used as the query; recall is the cosine of that query to its nearest
kept sentence. `1.0` means the answer survived compression. This is the
documented highest-value path (research / tool-result prose), and the place
prefix truncation fails structurally — it drops the back half wholesale.
The model must already be present (`available: false` is reported honestly when it
is not).
> Honest caveat: `docs/reference` is structured reference material whose first half
> is already representative, so on the **query-free `avg_coverage`** signal,
> prefix truncation is a *strong* baseline (often ahead) — we report that as
> measured rather than spin it. Extractive's structural win shows up in
> **`rag_query_recall`**: when the answer is not in the prefix, truncation cannot
> recover it and query-aware extraction can. Point `--corpus` at long,
> non-front-loaded prose / RAG dumps to widen the coverage gap too.
## Notes
- Prose/markdown is a **conservative** corpus for the rule-based funnel; tool
output, logs and RAG dumps (the common agent case) compress far more.
- lean-ctx output is deterministic and prompt-cache safe (#498); the benchmark
re-running with identical input yields identical `lean_ctx` figures.
See the full positioning in [docs/comparisons/vs-headroom.md](../../docs/comparisons/vs-headroom.md)
and the recipes in [docs/guides/compress-sdk.md](../../docs/guides/compress-sdk.md).
+179
View File
@@ -0,0 +1,179 @@
#!/usr/bin/env python3
"""Head-to-head compression benchmark: lean-ctx `/v1/compress` vs Headroom.
Runs both libraries over the *same* real corpus with the *same* tokenizer and
reports compression ratio + latency as JSON. Numbers are always measured, never
fabricated: a tool that is not installed/reachable is reported as
``available: false`` rather than guessed.
Prerequisites
-------------
* lean-ctx daemon with ``POST /v1/compress`` running (``lean-ctx dev-install``).
* Optional head-to-head: ``pip install headroom-ai``.
* Optional accurate token counts: ``pip install tiktoken`` (else char counts).
Usage
-----
python bench/compress/benchmark.py # JSON to stdout
python bench/compress/benchmark.py --corpus docs/ # custom corpus
python bench/compress/benchmark.py --out report.json --model gpt-4o
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
REPO_ROOT = Path(__file__).resolve().parents[2]
PY_SDK = REPO_ROOT / "packages" / "python-lean-ctx"
if PY_SDK.is_dir():
sys.path.insert(0, str(PY_SDK))
Message = Dict[str, Any]
def build_tokenizer(model: str) -> tuple[Callable[[str], int], str]:
"""Return ``(count_fn, name)``. Prefers tiktoken; falls back to chars."""
try:
import tiktoken
try:
enc = tiktoken.encoding_for_model(model)
except KeyError:
enc = tiktoken.get_encoding("o200k_base")
return (lambda text: len(enc.encode(text)), enc.name)
except Exception:
return (len, "chars")
def iter_text(content: Any):
"""Yield every text payload inside an OpenAI/Anthropic message content."""
if isinstance(content, str):
yield content
elif isinstance(content, list):
for block in content:
if not isinstance(block, dict):
continue
if block.get("type") == "text" and isinstance(block.get("text"), str):
yield block["text"]
elif block.get("type") == "tool_result":
yield from iter_text(block.get("content"))
def total_tokens(messages: List[Message], count: Callable[[str], int]) -> int:
return sum(count(text) for msg in messages for text in iter_text(msg.get("content")))
def load_corpus(path: Path, max_files: int, max_bytes: int) -> List[Message]:
"""Build one user message per real text file under ``path`` (no fixtures)."""
if not path.exists():
raise SystemExit(f"corpus path does not exist: {path}")
suffixes = {".md", ".rs", ".py", ".ts", ".txt", ".json", ".log", ".yaml", ".yml"}
files = sorted(p for p in path.rglob("*") if p.is_file() and p.suffix in suffixes)
messages: List[Message] = []
for file in files:
if len(messages) >= max_files:
break
try:
text = file.read_text(encoding="utf-8")
except (UnicodeDecodeError, OSError):
continue
if len(text) > max_bytes:
text = text[:max_bytes]
if text.strip():
messages.append({"role": "user", "content": text})
if not messages:
raise SystemExit(f"no readable text files found under {path}")
return messages
def measure(
label: str,
compress: Callable[[List[Message]], List[Message]],
messages: List[Message],
count: Callable[[str], int],
) -> Dict[str, Any]:
"""Run one compressor once, returning measured tokens + latency."""
original = total_tokens(messages, count)
started = time.perf_counter()
try:
out = compress(messages)
except Exception as exc: # noqa: BLE001 - any failure is reported, not raised
return {"available": False, "error": f"{type(exc).__name__}: {exc}"}
latency_ms = round((time.perf_counter() - started) * 1000, 2)
compressed = total_tokens(out, count)
ratio = round(1 - compressed / original, 4) if original else 0.0
return {
"available": True,
"original_tokens": original,
"compressed_tokens": compressed,
"tokens_saved": original - compressed,
"ratio": ratio,
"latency_ms": latency_ms,
}
def lean_ctx_compressor(model: str) -> Optional[Callable[[List[Message]], List[Message]]]:
try:
from lean_ctx import compress as lc_compress
except ImportError:
return None
return lambda messages: lc_compress(messages, model=model)
def headroom_compressor(model: str) -> Optional[Callable[[List[Message]], List[Message]]]:
try:
from headroom import compress as hr_compress
except ImportError:
return None
return lambda messages: hr_compress(messages, model=model).messages
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--corpus", default=str(REPO_ROOT / "docs" / "reference"))
parser.add_argument("--model", default="gpt-4o")
parser.add_argument("--max-files", type=int, default=50)
parser.add_argument("--max-bytes", type=int, default=200_000)
parser.add_argument("--out", help="write the JSON report to this file")
args = parser.parse_args()
count, tokenizer = build_tokenizer(args.model)
messages = load_corpus(Path(args.corpus), args.max_files, args.max_bytes)
report: Dict[str, Any] = {
"corpus": {
"path": str(Path(args.corpus)),
"messages": len(messages),
"model": args.model,
"tokenizer": tokenizer,
},
}
lc = lean_ctx_compressor(args.model)
report["lean_ctx"] = (
measure("lean-ctx", lc, messages, count)
if lc
else {"available": False, "install": "pip install lean-ctx-sdk (and run the daemon)"}
)
hr = headroom_compressor(args.model)
report["headroom"] = (
measure("headroom", hr, messages, count)
if hr
else {"available": False, "install": "pip install headroom-ai"}
)
payload = json.dumps(report, indent=2)
print(payload)
if args.out:
Path(args.out).write_text(payload + "\n", encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+251
View File
@@ -0,0 +1,251 @@
# lean-ctx — One Binary, Same Savings as 6 Tools Combined
**Response to: [Claude Code Token Savings Stack — 6 layers, zero overlap, ~60% context reduction](https://gist.github.com/doobidoo/e5500be6b59e47cadc39e0b7c5cd9871)**
What if one binary covers all 6 layers?
That's lean-ctx — a single Rust binary / MCP server that handles CLI compression, file read optimization, response compression, targeted context, and session knowledge. No 6 repos, no 3 programming languages, no 15-minute setup dance.
```
User Prompt
→ [lean-ctx ctx_knowledge] Cross-session memory → skip re-discovery
→ [lean-ctx ctx_read modes] Targeted context (map/signatures/aggressive) → skip full reads
→ LLM thinks → [CRP mode] Compact responses → no filler tokens
→ Tool calls → [lean-ctx MCP] 51 tools, compact schemas
→ Bash/CLI → [lean-ctx -c] 56 pattern modules → structured compression
→ File reads → [lean-ctx cache] Re-reads cost ~13 tokens
```
One install. One binary. Same result.
---
## Benchmark Setup
- **System:** macOS, lean-ctx 3.6.6
- **Project:** lean-ctx itself (Rust + TypeScript + Python, ~50K LOC)
- **Tokenizer:** tiktoken cl100k_base (GPT-4/Claude tokenizer, exact counts — not char/4 approximation)
- **Measurement:** Built-in `lean-ctx benchmark` command using Rust tiktoken bindings
- **Reproducible:** `lean-ctx benchmark run . --json` generates raw data
---
## Results: Layer-by-Layer Comparison
### 1. CLI Output Filtering (RTK equivalent)
lean-ctx intercepts shell commands via `lean-ctx -c` and applies 56 domain-specific compression modules (git, cargo, npm, docker, terraform, kubectl, etc.).
**Measured on 1,794 real commands (production usage):**
| Metric | lean-ctx | RTK |
|--------|----------|-----|
| Commands measured | 1,794 | 282 |
| Total input tokens | 91,344,503 | ~195K (estimated from 117K saved at 60%) |
| Tokens saved | 54,733,233 | 117,100 |
| **Savings rate** | **59.9%** | **60.2%** |
| Avoided cost (USD) | $136.83 | — |
**Per-command examples:**
| Command | Raw | Compressed | Savings |
|---------|-----|-----------|---------|
| `git log --stat -10` | 8,693 chars | 636 chars | 92.7% |
| `git diff HEAD~5 --stat` | 3,077 chars | 179 chars | 94.2% |
| `git log --oneline -50` | 3,431 chars | 1,221 chars | 64.4% |
| `git status` | 2,350 chars | 1,585 chars | 32.6% |
lean-ctx doesn't blindly filter — it pattern-matches structured output. Git stat blocks become one-liners, test results become summaries, verbose logs become actionable diffs.
---
### 2. File Read Compression (context-mode equivalent)
Instead of dumping raw files into context, lean-ctx auto-selects the optimal read mode per file.
**50 files measured across 9 languages (tiktoken exact counts):**
| Read Mode | Avg Savings | Quality Preserved | Use Case |
|-----------|-------------|-------------------|----------|
| `map` | 97.4% | 81% | Dependencies + API surface |
| `signatures` | 96.6% | 90% | Function/class signatures only |
| `cache_hit` | 99.8% | — | Re-reads from session cache |
| `aggressive` | 4.1% | 100% | Full content, comments stripped |
| `entropy` | 0.5% | 100% | Full content, high-entropy only |
**Per-language best savings:**
| Language | Files | Raw Tokens | Best Mode | Savings |
|----------|-------|-----------|-----------|---------|
| .rs | 10 | 144,295 | map | 96.5% |
| .md | 10 | 80,376 | aggressive | 5.6% |
| .js | 10 | 71,352 | map | 99.1% |
| .json | 5 | 67,430 | aggressive | 0.5% |
| .py | 9 | 26,688 | signatures | 94.5% |
| .css | 1 | 18,049 | aggressive | 2.4% |
| .ts | 4 | 13,974 | map | 95.6% |
| .html | 1 | 8,656 | aggressive | 2.4% |
> **Note on non-code files:** Markdown, JSON, CSS, HTML are data/markup files without code structures (functions, classes, types). lean-ctx's structural modes (`map`, `signatures`) extract code skeletons and are only applicable to programming languages. For data/markup files, only `aggressive` mode (whitespace/comment stripping) is reported. The high savings for code files (Rust 96.5%, Python 94.5%, JS 99.1%) come from extracting only the structural skeleton that an LLM needs for context.
**vs. context-mode:** context-mode sandboxes output into SQLite and returns BM25 snippets (~98% claimed). lean-ctx achieves 96-99% on code files through intelligent mode selection — no database, no indexing delay, deterministic results.
---
### 3. Tool Definition Size (MCPlex equivalent)
| Setup | Tools Exposed | Token Cost |
|-------|---------------|-----------|
| 6-tool stack (raw) | 37 tools | ~8,762 tokens |
| MCPlex gateway | 3 meta-tools | ~273 tokens |
| **lean-ctx** | **51 tools** | **~3,200 tokens** |
lean-ctx exposes all tools directly with compact JSON schemas. No gateway needed, no `find_tools()` indirection, no semantic routing overhead. The LLM sees all capabilities immediately.
**Trade-off:** MCPlex wins on raw token count (273 vs 3,200) by hiding tools. But lean-ctx tools are directly callable — no discovery round-trip needed, which saves 1-2 tool calls per interaction.
---
### 4. Response Compression (Caveman equivalent)
CRP (Compact Response Protocol) compresses tool responses in-flight:
| Mode | Tokens (30-min session) | Cost |
|------|------------------------|------|
| Raw (no lean-ctx) | 605,400 | $1.51 |
| lean-ctx | 84,400 | $0.21 |
| lean-ctx + CRP | 79,900 | $0.20 |
**CRP savings over lean-ctx alone: additional ~5.4% compression** through abbreviations, delta-only diffs, and structured `+/-/~` notation.
**vs. Caveman (20-40% on output):** CRP operates at the tool-output level, not the LLM response level. They're complementary — you could use both. But lean-ctx's modes already deliver the bigger wins upstream.
---
### 5. Targeted Context (MCP-Context-Provider equivalent)
Instead of a separate server providing context rules, lean-ctx's 10 read modes ARE the targeted context:
```
Developer asks: "How does auth work?"
Without lean-ctx:
→ Read auth.rs (full) = 2,500 tokens
→ Read middleware.rs (full) = 1,800 tokens
→ Read config.rs (full) = 900 tokens
Total: 5,200 tokens
With lean-ctx (auto-mode):
→ ctx_read auth.rs mode=map = 65 tokens (deps + API)
→ ctx_read middleware.rs mode=map = 42 tokens
→ ctx_read config.rs mode=map = 18 tokens
Total: 125 tokens (97.6% less)
```
No separate service. No rule configuration. The compression IS the context targeting.
---
### 6. Session Knowledge (MCP-Memory-Service equivalent)
lean-ctx provides cross-session persistence without a vector database:
| Feature | lean-ctx | MCP-Memory-Service |
|---------|----------|-------------------|
| Cross-session memory | `ctx_knowledge remember/recall` | `memory_store/search` |
| Session state | `ctx_session` (auto-compaction) | — |
| Re-read cost | ~13 tokens (cached) | N/A |
| Warm start | `ctx_preload` | Embedding search |
| Storage | Local files (instant) | SQLite + Cloudflare Vectorize |
| Setup | Zero config | API tokens, cloud setup |
**Measured re-read savings:**
- First read of 10 source files: ~15,000 tokens
- Re-read (session cache): ~130 tokens (10 × ~13 tok)
- Knowledge recall: ~200-500 tokens
**Effective savings: 95-99% on repeated access.**
---
## Session Simulation: Combined Savings
**30-minute coding session (50 files, multiple reads, shell commands):**
| Setup | Tokens | Cost | Savings |
|-------|--------|------|---------|
| Raw (no compression) | 605,400 | $1.51 | — |
| lean-ctx (all modes) | 84,400 | $0.21 | **86.1%** |
| lean-ctx + CRP | 79,900 | $0.20 | **86.8%** |
**The 6-tool stack claims ~58.5% savings. lean-ctx measured 86.1-86.8%.**
---
## Why the Difference?
The 6-tool stack operates at different layers that don't compose perfectly. lean-ctx is architecturally integrated:
1. **No inter-tool overhead** — One process, one cache, one tokenizer
2. **Mode selection is aware of context** — The cache knows what was already sent
3. **Re-reads are essentially free** — Session-aware caching eliminates redundant I/O
4. **Shell + file reads compound** — The same session state optimizes both
---
## Methodology & Transparency
- **Token counting:** Rust bindings to tiktoken (cl100k_base) — exact token counts, not char/4 approximation
- **"Best mode" selection:** Only modes that produce meaningful output qualify. A mode returning 0 tokens (e.g., `map` on JSON) is excluded — that's data loss, not compression
- **Quality score:** Semantic preservation measured via key-symbol retention (exported names, types, function signatures)
- **Reproducibility:** Run `lean-ctx benchmark run /your/project --json` on any codebase
- **Not cherry-picked:** Benchmark runs on ALL files matching configured extensions, not hand-selected examples
---
## Install
```bash
# One command. 30 seconds. Done.
cargo install lean-ctx
# Or from source:
git clone https://github.com/yvgude/lean-ctx
cd lean-ctx/rust && cargo build --release
```
vs. the 6-tool stack:
```bash
# 6 repos, 3 languages, 15 minutes, bridge configs...
cargo install rtk
git clone mcplex && cargo build
git clone MCP-Context-Provider && npm install && npm run build
git clone mcp-memory-service && uv sync
# + Claude Code plugin installs
# + MCPlex upstream configuration
# + bridge.mjs for macOS...
```
---
## Reproduce This Benchmark
```bash
# After installing lean-ctx:
lean-ctx benchmark run . # Human-readable output
lean-ctx benchmark run . --json # Raw JSON data (per-file, per-mode, tiktoken counts)
lean-ctx gain # CLI compression stats (cumulative production usage)
lean-ctx gain --json # CLI stats as JSON
```
---
## Links
- **GitHub:** [github.com/yvgude/lean-ctx](https://github.com/yvgude/lean-ctx)
- **Install:** `cargo install lean-ctx`
- **Version:** 3.6.6
---
*Measured 2026-05-18 on lean-ctx 3.6.6 against the lean-ctx codebase itself (Rust/TS/Python, ~50K LOC). All token counts from tiktoken cl100k_base (exact), not character-based estimates. Benchmark fix applied: modes returning 0 tokens excluded from "best" ranking — 0 output is data loss, not compression.*
+25
View File
@@ -0,0 +1,25 @@
# LoCoMo Memory Benchmark — lean-ctx
Suite: `reference-suite` · samples: 2 · questions: 13 · top_k: 5
Retrieval-recall benchmark: each conversation turn is stored as a memory, then for every question the top-k memories are recalled and scored against the gold answers. Model-free and deterministic.
## Overall
| metric | value |
|---|---|
| answer containment (recall@5) | 100.0% |
| mean best-memory token-F1 | 0.229 |
| exact-match rate | 0.0% |
| mean recalled-context tokens | 82 |
| mean full-transcript tokens | 116 |
| token reduction vs. full transcript | 29.4% |
## By category
| category | questions | containment | mean F1 | recall tokens |
|---|---|---|---|---|
| single-hop | 11 | 100.0% | 0.232 | 80 |
| temporal | 2 | 100.0% | 0.213 | 95 |
_Generated 2026-06-09T06:50:06.284618+00:00._
@@ -0,0 +1,39 @@
{
"suite": "reference-suite",
"generated_at": "2026-06-09T06:50:06.284618+00:00",
"top_k": 5,
"samples": 2,
"questions": 13,
"overall": {
"category": 0,
"label": "overall",
"questions": 13,
"containment_rate": 1.0,
"mean_f1": 0.229,
"exact_match_rate": 0.0,
"mean_recall_tokens": 81.923
},
"by_category": [
{
"category": 1,
"label": "single-hop",
"questions": 11,
"containment_rate": 1.0,
"mean_f1": 0.232,
"exact_match_rate": 0.0,
"mean_recall_tokens": 79.545
},
{
"category": 3,
"label": "temporal",
"questions": 2,
"containment_rate": 1.0,
"mean_f1": 0.213,
"exact_match_rate": 0.0,
"mean_recall_tokens": 95.0
}
],
"mean_transcript_tokens": 116.0,
"mean_recall_tokens": 81.923,
"token_reduction_pct": 29.377
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+144
View File
@@ -0,0 +1,144 @@
{
"version": "2.1",
"lean_ctx_version": "3.6.6",
"date": "2026-05-18T13:00:00Z",
"tokenizer": "tiktoken cl100k_base (exact via Rust bindings)",
"project": "lean-ctx (Rust/TS/Python, ~50K LOC)",
"cli_compression": {
"source": "production usage (lean-ctx gain)",
"total_commands": 1973,
"input_tokens": 99711868,
"output_tokens": 39983946,
"tokens_saved": 59727922,
"savings_pct": 59.9,
"avoided_usd": 149.32
},
"file_read_compression": {
"files_measured": 50,
"total_raw_tokens": 430767,
"languages": [
{
"best_mode": "map",
"best_mode_tokens": 5016,
"best_savings_pct": 96.52,
"count": 10,
"ext": "rs",
"total_tokens": 144295
},
{
"best_mode": "aggressive",
"best_mode_tokens": 75846,
"best_savings_pct": 5.64,
"count": 10,
"ext": "md",
"total_tokens": 80376
},
{
"best_mode": "map",
"best_mode_tokens": 634,
"best_savings_pct": 99.11,
"count": 10,
"ext": "js",
"total_tokens": 71352
},
{
"best_mode": "aggressive",
"best_mode_tokens": 67045,
"best_savings_pct": 0.47,
"count": 5,
"ext": "json",
"total_tokens": 67359
},
{
"best_mode": "signatures",
"best_mode_tokens": 1462,
"best_savings_pct": 94.53,
"count": 9,
"ext": "py",
"total_tokens": 26706
},
{
"best_mode": "aggressive",
"best_mode_tokens": 17615,
"best_savings_pct": 2.4,
"count": 1,
"ext": "css",
"total_tokens": 18049
},
{
"best_mode": "map",
"best_mode_tokens": 612,
"best_savings_pct": 95.62,
"count": 4,
"ext": "ts",
"total_tokens": 13974
},
{
"best_mode": "aggressive",
"best_mode_tokens": 8450,
"best_savings_pct": 2.38,
"count": 1,
"ext": "html",
"total_tokens": 8656
}
],
"mode_summaries": [
{
"avg_latency_us": 6050,
"avg_preservation": 81.16,
"avg_savings_pct": 97.59,
"mode": "map",
"total_compressed_tokens": 7864
},
{
"avg_latency_us": 2308,
"avg_preservation": 91.0,
"avg_savings_pct": 96.82,
"mode": "signatures",
"total_compressed_tokens": 10992
},
{
"avg_latency_us": 136,
"avg_preservation": 100.0,
"avg_savings_pct": 3.91,
"mode": "aggressive",
"total_compressed_tokens": 414596
},
{
"avg_latency_us": 12901,
"avg_preservation": 99.89,
"avg_savings_pct": 0.5,
"mode": "entropy",
"total_compressed_tokens": 428940
},
{
"avg_latency_us": 0,
"avg_preservation": null,
"avg_savings_pct": 99.76,
"mode": "cache_hit",
"total_compressed_tokens": 650
}
]
},
"session_simulation": {
"ccp_cost_usd": 0.2,
"lean_ccp_tokens": 79868,
"lean_cost_usd": 0.21,
"lean_tokens": 84429,
"raw_cost_usd": 1.51,
"raw_tokens": 605422
},
"methodology": {
"token_counting": "tiktoken cl100k_base via Rust bindings (exact counts, not char/4)",
"best_mode_selection": "Structural modes (map, signatures) only for code files. Data/markup files (md, json, css, html) limited to aggressive/entropy. Modes returning 0 tokens always excluded.",
"quality_metric": "Semantic preservation via key-symbol retention (exported names, types, signatures)",
"cli_measurement": "Production telemetry from 1,962 real commands over multiple weeks",
"reproducibility": "lean-ctx benchmark run /path --json",
"honesty_notes": [
"Markdown 5.6% (not 99.9%): md has no code structures, only whitespace stripping applies",
"JSON 0.5%: no compression beyond whitespace normalization",
"CSS/HTML 2.4%: comment removal only",
"High savings (94-99%) only for actual programming languages with parseable structures"
]
}
}
+185
View File
@@ -0,0 +1,185 @@
#!/usr/bin/env bash
set -euo pipefail
# lean-ctx Benchmark v2.0
# Uses lean-ctx's built-in tiktoken tokenizer for exact measurements.
# No char/4 approximations.
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
RESULTS_DIR="$SCRIPT_DIR/results"
LEAN_CTX="${LEAN_CTX_BIN:-lean-ctx}"
mkdir -p "$RESULTS_DIR"
echo "═══════════════════════════════════════════════════"
echo " lean-ctx Benchmark v2.0"
echo " Project: $PROJECT_ROOT"
echo " Binary: $($LEAN_CTX --version 2>/dev/null || echo 'unknown')"
echo " Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "═══════════════════════════════════════════════════"
echo ""
# ═══════════════════════════════════════════════════════
# CATEGORY 1: CLI Output Filtering (vs. RTK)
# Uses lean-ctx's built-in token counter for accuracy
# ═══════════════════════════════════════════════════════
echo "╔═══ Category 1: CLI Output Filtering ═══╗"
echo ""
declare -a CLI_COMMANDS=(
"git log --oneline -20"
"git status"
"git diff HEAD~3 --stat"
"git branch -a"
"git log --format='%h %s %an %ar' -30"
"ls -la"
"ls -laR rust/src/core/patterns/"
"cargo test --lib -- --list 2>&1 | head -100"
"cargo tree --depth 1"
"git log --stat -5"
"git shortlog -sn --all"
"git remote -v"
"cat Cargo.toml"
"cat rust/Cargo.toml"
"env | sort | head -40"
"ps aux | head -30"
"df -h"
)
CLI_TOTAL_RAW=0
CLI_TOTAL_COMPRESSED=0
CLI_COUNT=0
for cmd in "${CLI_COMMANDS[@]}"; do
# Get raw output
RAW_OUTPUT=$(cd "$PROJECT_ROOT" && eval "$cmd" 2>/dev/null || true)
if [ -z "$RAW_OUTPUT" ] || [ ${#RAW_OUTPUT} -lt 10 ]; then
continue
fi
# Count tokens via lean-ctx's tokenizer (pipe to token count)
RAW_TOKENS=$(echo "$RAW_OUTPUT" | "$LEAN_CTX" tokens count 2>/dev/null || echo "$RAW_OUTPUT" | wc -m | awk '{print int($1/3.5)}')
# Get compressed output (suppress savings footer)
COMPRESSED_OUTPUT=$(cd "$PROJECT_ROOT" && LEAN_CTX_SAVINGS_FOOTER=0 "$LEAN_CTX" -c "$cmd" 2>/dev/null || true)
COMPRESSED_TOKENS=$(echo "$COMPRESSED_OUTPUT" | "$LEAN_CTX" tokens count 2>/dev/null || echo "$COMPRESSED_OUTPUT" | wc -m | awk '{print int($1/3.5)}')
if [ "$RAW_TOKENS" -gt 10 ]; then
CLI_TOTAL_RAW=$((CLI_TOTAL_RAW + RAW_TOKENS))
CLI_TOTAL_COMPRESSED=$((CLI_TOTAL_COMPRESSED + COMPRESSED_TOKENS))
CLI_COUNT=$((CLI_COUNT + 1))
SAVINGS_PCT=$(echo "scale=1; ($RAW_TOKENS - $COMPRESSED_TOKENS) * 100 / $RAW_TOKENS" | bc 2>/dev/null || echo "0")
printf " [%2d] %-45s %6s → %6s (-%s%%)\n" "$CLI_COUNT" "$cmd" "$RAW_TOKENS" "$COMPRESSED_TOKENS" "$SAVINGS_PCT"
fi
done
CLI_SAVINGS_PCT=$(echo "scale=1; ($CLI_TOTAL_RAW - $CLI_TOTAL_COMPRESSED) * 100 / $CLI_TOTAL_RAW" | bc 2>/dev/null || echo "0")
echo ""
echo " ── CLI Summary: ${CLI_COUNT} commands"
echo " Total raw: ${CLI_TOTAL_RAW} tokens"
echo " Total compressed: ${CLI_TOTAL_COMPRESSED} tokens"
echo " Savings: ${CLI_SAVINGS_PCT}%"
echo ""
# ═══════════════════════════════════════════════════════
# CATEGORY 2: File Read Compression
# Uses lean-ctx's built-in benchmark (tiktoken exact)
# ═══════════════════════════════════════════════════════
echo "╔═══ Category 2: File Read Compression (tiktoken exact) ═══╗"
echo ""
# Run built-in benchmark with JSON output
BENCHMARK_JSON=$(cd "$PROJECT_ROOT" && "$LEAN_CTX" benchmark run . --json 2>/dev/null)
if [ -z "$BENCHMARK_JSON" ]; then
echo " ERROR: lean-ctx benchmark returned empty output"
exit 1
fi
# Save raw benchmark JSON
echo "$BENCHMARK_JSON" > "$RESULTS_DIR/benchmark-raw.json"
# Display results
echo "$BENCHMARK_JSON" | python3 -c "
import json, sys
d = json.load(sys.stdin)
print(f' Files measured: {d[\"files_measured\"]}')
print(f' Total raw tokens: {d[\"total_raw_tokens\"]:,}')
print()
print(' Per-language results:')
print(f' {\"Lang\":<8s} {\"Files\":>5s} {\"Raw Tok\":>9s} {\"Best Mode\":<12s} {\"Compressed\":>10s} {\"Savings\":>8s}')
print(f' {\"─\"*8} {\"─\"*5} {\"─\"*9} {\"─\"*12} {\"─\"*10} {\"─\"*8}')
for l in d.get('languages', []):
print(f' {l[\"ext\"]:<8s} {l[\"count\"]:>5d} {l[\"total_tokens\"]:>9,} {l[\"best_mode\"]:<12s} {l[\"best_mode_tokens\"]:>10,} {l[\"best_savings_pct\"]:>7.1f}%')
print()
print(' Mode performance:')
for ms in d.get('mode_summaries', []):
q = f'{ms[\"avg_preservation\"]:.0f}%' if ms.get('avg_preservation', 0) > 0 else 'N/A'
print(f' {ms[\"mode\"]:15s} savings: {ms[\"avg_savings_pct\"]:>5.1f}% quality: {q:>5s}')
"
echo ""
# ═══════════════════════════════════════════════════════
# SESSION SIMULATION
# ═══════════════════════════════════════════════════════
echo "╔═══ Session Simulation (30-min coding) ═══╗"
echo ""
# Display human-readable benchmark (includes session sim)
cd "$PROJECT_ROOT" && "$LEAN_CTX" benchmark run . 2>/dev/null | grep -A10 "Session Simulation"
echo ""
# ═══════════════════════════════════════════════════════
# CUMULATIVE PRODUCTION STATS
# ═══════════════════════════════════════════════════════
echo "╔═══ Cumulative Production Stats ═══╗"
echo ""
cd "$PROJECT_ROOT" && "$LEAN_CTX" gain 2>/dev/null || echo " (no production data available)"
echo ""
# ═══════════════════════════════════════════════════════
# SAVE FINAL COMBINED RESULTS
# ═══════════════════════════════════════════════════════
# Combine CLI + benchmark into final JSON
echo "$BENCHMARK_JSON" | python3 -c "
import json, sys
d = json.load(sys.stdin)
results = {
'version': '2.0',
'lean_ctx_version': '3.6.6',
'date': '$(date -u +%Y-%m-%dT%H:%M:%SZ)',
'tokenizer': 'tiktoken cl100k_base (exact)',
'project': 'lean-ctx (Rust/TS/Python, ~50K LOC)',
'cli_compression': {
'commands_tested': $CLI_COUNT,
'raw_tokens': $CLI_TOTAL_RAW,
'compressed_tokens': $CLI_TOTAL_COMPRESSED,
'savings_pct': round(($CLI_TOTAL_RAW - $CLI_TOTAL_COMPRESSED) * 100 / max($CLI_TOTAL_RAW, 1), 1)
},
'file_read_compression': {
'files_measured': d['files_measured'],
'total_raw_tokens': d['total_raw_tokens'],
'languages': d.get('languages', []),
'mode_summaries': d.get('mode_summaries', [])
},
'session_simulation': d.get('session_simulation', {}),
'methodology': {
'token_counting': 'tiktoken cl100k_base via Rust bindings (exact counts)',
'best_mode_selection': 'Only modes producing >0 tokens qualify (0 = data loss, excluded)',
'quality_metric': 'Semantic preservation via key-symbol retention',
'reproducibility': 'lean-ctx benchmark run /path --json'
}
}
print(json.dumps(results, indent=2))
" > "$RESULTS_DIR/benchmark-results.json"
echo "═══════════════════════════════════════════════════"
echo " Results saved to: $RESULTS_DIR/benchmark-results.json"
echo " Raw data: $RESULTS_DIR/benchmark-raw.json"
echo "═══════════════════════════════════════════════════"
+58
View File
@@ -0,0 +1,58 @@
# lean-ctx Efficiency Benchmark — Methodology
Reproducible harness for the **lean-ctx Efficiency Epic**. It proves every phase
on two axes that actually matter for an agent loop: **wall-time latency**
(p50/p95/p99) and **total response tokens** (`tiktoken`, via the same counter
the tools use), measured MCP-resident (the way the agent actually pays for it).
## Why these axes
Per-call byte savings can be misleading: aggressive compression that shaves a
few bytes per response often forces the agent into extra reads, raising **total
task tokens and tool-calls**. The optimization target is therefore **total task
tokens + wall-time**, never per-call bytes. The harness measures exactly that.
## Latency + tokens (runnable now)
Custom Rust harness (`rust/benches/efficiency.rs`, `harness = false`):
```bash
cd rust
cargo bench --bench efficiency # 2000-file synthetic corpus
BENCH_FILES=6000 cargo bench --bench efficiency # react-scale corpus
```
It builds a deterministic synthetic corpus (files across 20 dirs, a common
token, a rare camelCase token, and a guaranteed-absent negative query), warms
once, then runs `ITERS=50` and reports `p50/p95/p99` ms plus response tokens.
From Phase 1 on, the harness emits two blocks — **Walk path (legacy)** and
**Resident index** — on the same corpus and queries, so the speedup is visible
in a single run with no "before" git checkout.
### Corpora
- **self** — the lean-ctx Rust tree (`rust/`), real-world mixed file sizes.
- **synthetic-2000 / synthetic-6000** — deterministic, react-scale, CI-stable.
## Agentic mini-eval (protocol)
The latency/token harness cannot exercise an LLM loop, so the agentic axis is a
documented protocol run against a real model with the MCP server attached. Use
3-4 natural-language tasks per corpus and record **tool-calls, wall-time, total
tokens, quality (pass/fail rubric)**:
1. "Where is the search index built and how does ctx_search use it?"
2. "Find the function that flushes passive effects and show its body."
3. "Add a parameter to the BM25 cache TTL and list every call site."
4. "Trace how a provider result reaches the BM25 index."
Freeze the baseline numbers in `RESULTS-baseline.md`, then re-run after each
phase and diff. Acceptance for a phase is **fewer-or-equal tool-calls and
total tokens at equal-or-better quality** vs. the frozen baseline.
## Recall parity (Phase 1 gate)
The resident index must not change *which* lines `ctx_search` returns. The
harness asserts set-equality of `file:line` hits between the walk path and the
index path (Jaccard ≥ 0.95) on every query before reporting latency.
+56
View File
@@ -0,0 +1,56 @@
# lean-ctx Efficiency — Baseline + Phase 1 Results
Frozen reference for the efficiency epic. Reproduce with:
```bash
cd rust && cargo bench --bench efficiency # 2000-file synthetic corpus
```
The harness measures the **walk path** (index forced off via
`LEAN_CTX_DISABLE_SEARCH_INDEX=1`) and the **resident index path** (warmed
synchronously) on the same corpus and queries, asserting recall parity
(identical `file:line` hits, Jaccard = 1.0) before reporting latency.
## `ctx_search` — synthetic-2000 (2000 files, 50 iters)
### Walk path (legacy baseline)
| query | p50 ms | p95 ms | p99 ms | resp tokens |
|---|---|---|---|---|
| common (`handler`) | 4.649 | 4.953 | 5.806 | 427 |
| rare (`flushPassiveEffectsRare`) | 25.841 | 27.944 | 28.506 | 108 |
| negative (`xyzzy_nonexistent`) | 26.153 | 27.762 | 28.743 | 14 |
### Resident index path (Phase 1)
| query | p50 ms | p95 ms | p99 ms | resp tokens |
|---|---|---|---|---|
| common (`handler`) | 0.265 | 0.386 | 0.483 | 427 |
| rare (`flushPassiveEffectsRare`) | 0.088 | 0.103 | 0.116 | 107 |
| negative (`xyzzy_nonexistent`) | 0.026 | 0.029 | 0.031 | 13 |
### Speedup (p50)
| query | walk | index | speedup |
|---|---|---|---|
| common | 4.649 ms | 0.265 ms | **17.5×** |
| rare | 25.841 ms | 0.088 ms | **293×** |
| negative | 26.153 ms | 0.026 ms | **~1000×** |
## Interpretation
- **Phase 1 acceptance met.** p50 (warm) drops well below the 5 ms target on
every query; recall is byte-identical (parity assertion passes, `warm=true`).
- The rare/negative queries gain the most: trigram narrowing reads ~0 candidate
files instead of all 2000. A negative query short-circuits to `O(1)` (a
required trigram is absent from the index).
- Response token counts are unchanged — the win is pure latency, no behavioral
change to what the agent receives. (The 1-token deltas on the no-match
messages are the "N files searched" count: candidates vs. full corpus.)
## Notes on the token/agentic axis
Per-call response tokens are intentionally unchanged in Phase 1 (latency-only).
The total-task-token and tool-call reductions come from Phases 2-4
(`ctx_compose`, inline read bodies, symbol-map default-off) and are measured via
the agentic mini-eval protocol in `METHODOLOGY.md`.
Executable
+222
View File
@@ -0,0 +1,222 @@
#!/usr/bin/env bash
# lctx — lean-ctx launcher for AI coding agents
#
# Usage:
# lctx # auto-detect agent, current dir
# lctx --agent claude # start Claude Code
# lctx --agent cursor # configure Cursor
# lctx --agent gemini # start Gemini CLI
# lctx --agent codex # configure Codex
# lctx --agent windsurf # configure Windsurf
# lctx --scan-only # build graph only
# lctx /path/to/project # specific project
# lctx /path/to/project "prompt" # with initial prompt (Claude only)
# lctx --resume ID # resume Claude session
set -euo pipefail
LEAN_CTX=""
AGENT=""
AGENTS=""
PROJECT_DIR=""
PROMPT=""
RESUME_ID=""
SCAN_ONLY=false
PARALLEL=false
find_lean_ctx() {
if command -v lean-ctx &>/dev/null; then
LEAN_CTX="$(command -v lean-ctx)"
elif [[ -x "${SCRIPT_DIR}/../rust/target/release/lean-ctx" ]]; then
LEAN_CTX="${SCRIPT_DIR}/../rust/target/release/lean-ctx"
else
echo "Error: lean-ctx not found."
echo "Install: cargo install lean-ctx"
echo " or: curl -fsSL https://raw.githubusercontent.com/yvgude/lean-ctx/main/install.sh | bash -s -- --download"
exit 1
fi
}
detect_agent() {
if [[ -n "$AGENT" ]]; then return; fi
if command -v claude &>/dev/null && [[ -d "$HOME/.claude" ]]; then
AGENT="claude"
elif command -v codebuddy &>/dev/null || [[ -d "$HOME/.codebuddy" ]]; then
AGENT="codebuddy"
elif [[ -d "$HOME/.cursor" ]]; then
AGENT="cursor"
elif command -v gemini &>/dev/null || [[ -d "$HOME/.gemini" ]]; then
AGENT="gemini"
elif command -v codex &>/dev/null || [[ -d "$HOME/.codex" ]]; then
AGENT="codex"
else
AGENT="claude"
fi
echo "Auto-detected agent: $AGENT"
}
find_project_root() {
if [[ -n "$PROJECT_DIR" ]]; then return; fi
if git rev-parse --show-toplevel &>/dev/null 2>&1; then
PROJECT_DIR="$(git rev-parse --show-toplevel)"
else
PROJECT_DIR="$(pwd)"
fi
}
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
while [[ $# -gt 0 ]]; do
case "$1" in
--agent|-a)
AGENT="$2"; shift 2 ;;
--agents)
AGENTS="$2"; PARALLEL=true; shift 2 ;;
--resume|-r)
RESUME_ID="$2"; shift 2 ;;
--scan-only)
SCAN_ONLY=true; shift ;;
--help|-h)
echo "Usage: lctx [options] [project_dir] [prompt]"
echo ""
echo "Options:"
echo " --agent, -a NAME Agent to use: claude|codebuddy|cursor|gemini|codex|windsurf|cline"
echo " --agents LIST Launch multiple agents in parallel (comma-separated)"
echo " --resume, -r ID Resume Claude Code session"
echo " --scan-only Build project graph only, don't launch agent"
echo " --help, -h Show this help"
echo ""
echo "Examples:"
echo " lctx # auto-detect, current dir"
echo " lctx --agent claude ~/my-project # Claude Code on specific project"
echo ' lctx ~/my-project "fix auth" # Claude Code with prompt'
echo " lctx --agents claude,gemini # parallel agents"
echo " lctx --scan-only # just build the graph"
exit 0 ;;
-*)
echo "Unknown option: $1"; exit 1 ;;
*)
if [[ -z "$PROJECT_DIR" && -d "$1" ]]; then
PROJECT_DIR="$(cd "$1" && pwd)"
elif [[ -z "$PROMPT" ]]; then
PROMPT="$1"
else
PROMPT="$PROMPT $1"
fi
shift ;;
esac
done
find_lean_ctx
find_project_root
echo "Project: $PROJECT_DIR"
echo "Using: $LEAN_CTX"
echo ""
echo "Building project graph..."
"$LEAN_CTX" graph build "$PROJECT_DIR" 2>/dev/null \
|| echo "(graph build skipped — will be built automatically in MCP mode)"
if $SCAN_ONLY; then
echo "Done (scan only)."
exit 0
fi
if $PARALLEL; then
echo "Multi-Agent Launch: $AGENTS"
echo ""
IFS=',' read -ra AGENT_LIST <<< "$AGENTS"
PIDS=()
for ag in "${AGENT_LIST[@]}"; do
ag="$(echo "$ag" | xargs)"
echo "Starting agent: $ag"
(
export LEAN_CTX_AGENT_ROLE="$ag"
"$0" --agent "$ag" "$PROJECT_DIR" ${PROMPT:+"$PROMPT"}
) &
PIDS+=($!)
done
echo ""
echo "Launched ${#AGENT_LIST[@]} agents in parallel (PIDs: ${PIDS[*]})"
echo "Use 'lean-ctx agent sync' or ctx_agent(action=sync) to monitor."
wait
exit 0
fi
detect_agent
echo ""
case "$AGENT" in
claude|claude-code)
echo "Setting up lean-ctx for Claude Code..."
"$LEAN_CTX" init --agent claude --global 2>/dev/null || true
CLAUDE_ARGS=()
if [[ -n "$RESUME_ID" ]]; then
CLAUDE_ARGS+=("--resume" "$RESUME_ID")
elif [[ -n "$PROMPT" ]]; then
CLAUDE_ARGS+=("-p" "$PROMPT")
fi
echo "Launching Claude Code..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
cd "$PROJECT_DIR"
exec claude ${CLAUDE_ARGS[@]+"${CLAUDE_ARGS[@]}"} ;;
codebuddy)
echo "Setting up lean-ctx for CodeBuddy..."
"$LEAN_CTX" init --agent codebuddy --global 2>/dev/null || true
CODEBUDDY_ARGS=()
if [[ -n "$RESUME_ID" ]]; then
CODEBUDDY_ARGS+=("--resume" "$RESUME_ID")
elif [[ -n "$PROMPT" ]]; then
CODEBUDDY_ARGS+=("-p" "$PROMPT")
fi
echo "Launching CodeBuddy..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
cd "$PROJECT_DIR"
exec codebuddy ${CODEBUDDY_ARGS[@]+"${CODEBUDDY_ARGS[@]}"} ;;
cursor)
echo "Setting up lean-ctx for Cursor..."
"$LEAN_CTX" init --agent cursor --global 2>/dev/null || true
echo ""
echo "Cursor configured. Open your project in Cursor to use lean-ctx MCP tools."
echo " cursor $PROJECT_DIR" ;;
gemini)
echo "Setting up lean-ctx for Gemini CLI..."
"$LEAN_CTX" init --agent gemini --global 2>/dev/null || true
echo "Launching Gemini CLI..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
cd "$PROJECT_DIR"
exec gemini ;;
codex)
echo "Setting up lean-ctx for Codex..."
"$LEAN_CTX" init --agent codex --global 2>/dev/null || true
echo ""
echo "Codex configured. lean-ctx hooks installed." ;;
windsurf)
echo "Setting up lean-ctx for Windsurf..."
"$LEAN_CTX" init --agent windsurf 2>/dev/null || true
echo ""
echo "Windsurf configured. Open your project in Windsurf." ;;
cline|roo)
echo "Setting up lean-ctx for Cline/Roo..."
"$LEAN_CTX" init --agent cline 2>/dev/null || true
echo ""
echo "Cline configured." ;;
*)
echo "Unknown agent: $AGENT"
echo "Supported: claude, cursor, gemini, codex, windsurf, cline"
exit 1 ;;
esac
+296
View File
@@ -0,0 +1,296 @@
# Building a Context Runtime for AI Coding Agents in Rust
lean-ctx is an open-source context runtime that sits between AI coding tools (Cursor, Claude Code, Copilot, etc.) and the filesystem. It compresses file reads with AST-aware intelligence, strips noise from shell output via 95+ patterns, and manages cross-session memory. A single Rust binary, 76 MCP tools, zero runtime dependencies.
This post walks through the Rust-specific architecture decisions, patterns, and trade-offs that shaped lean-ctx — from tree-sitter integration to implementing Thompson Sampling bandits without pulling in a statistics crate.
GitHub: [github.com/yvgude/lean-ctx](https://github.com/yvgude/lean-ctx) | Website: [leanctx.com](https://leanctx.com)
## The problem lean-ctx solves
AI coding agents read the same files repeatedly. A typical session reads `main.rs` ten to fifteen times at roughly 2,000 tokens each. Shell commands like `cargo build` or `git log` produce verbose output that burns through context windows. lean-ctx intercepts these reads and shell outputs, caching, compressing, and deduplicating before the LLM ever sees them.
The performance constraint is strict: lean-ctx sits in the hot path of every tool call. Responses must complete in under 50ms. That ruled out any approach involving local LLM inference for summarization and pushed us toward deterministic, algorithmic compression — which turned out to be both faster and more reliable.
## Tree-sitter: two integration styles in one codebase
lean-ctx supports AST-aware parsing for 18 languages. We use tree-sitter in two distinct patterns, both gated behind `#[cfg(feature = "tree-sitter")]` to keep builds without grammar dependencies fast and small.
**Pattern 1: Manual node walking.** The `deep_queries` module traverses the AST directly, dispatching per language extension:
```rust
pub fn analyze(content: &str, ext: &str) -> DeepAnalysis {
#[cfg(feature = "tree-sitter")]
{
if let Some(result) = analyze_with_tree_sitter(content, ext) {
return result;
}
}
DeepAnalysis::empty()
}
#[cfg(feature = "tree-sitter")]
fn analyze_with_tree_sitter(content: &str, ext: &str) -> Option<DeepAnalysis> {
let language = get_language(ext)?;
let mut parser = Parser::new();
parser.set_language(&language).ok()?;
let tree = parser.parse(content.as_bytes(), None)?;
let root = tree.root_node();
Some(DeepAnalysis {
imports: imports::extract_imports(root, content, ext),
calls: calls::extract_calls(root, content, ext),
types: type_defs::extract_types(root, content, ext),
exports: type_defs::extract_exports(root, content, ext),
})
}
```
The `?` operator chains neatly through tree-sitter's fallible steps — language lookup, parser configuration, parsing — collapsing any failure to `None` and falling back to `empty()`.
**Pattern 2: Declarative queries with a thread-local parser.** The `signatures_ts` module uses tree-sitter's query language to extract function and type signatures. Since `Parser` is not `Send`, we store it in a `thread_local!`:
```rust
thread_local! {
static PARSER: std::cell::RefCell<Parser> = std::cell::RefCell::new(Parser::new());
}
let tree = PARSER.with(|p| {
let mut parser = p.borrow_mut();
let _ = parser.set_language(&language);
parser.parse(content, None)
})?;
let query = Query::new(&language, query_src).ok()?;
let def_idx = find_capture_index(&query, "def")?;
let name_idx = find_capture_index(&query, "name")?;
let mut cursor = QueryCursor::new();
let mut matches = cursor.matches(&query, tree.root_node(), content.as_bytes());
while let Some(m) = matches.next() {
// extract captures by index, not magic numbers
}
```
One lesson learned: tree-sitter grammar crates don't always keep pace with tree-sitter core. We had to use `tree-sitter-kotlin-ng` instead of `tree-sitter-kotlin` because the original crate caps tree-sitter at <0.23, while we needed 0.26. This kind of version alignment challenge is common when pulling in many optional grammar dependencies.
## Adaptive compression with Thompson Sampling
Different files benefit from different compression levels. A 50-line utility file shouldn't be compressed the same way as a 2,000-line service module. Rather than hard-coding thresholds, lean-ctx learns them during each session using a multi-armed bandit.
Each "arm" represents a compression threshold. The bandit selects which threshold to apply, then observes whether the AI agent needed to re-read the file (a signal that compression was too aggressive). Over a session, the bandit converges on the threshold that balances token savings against information loss.
The core selection uses epsilon-greedy exploration with a decaying epsilon, combined with Beta-distributed Thompson Sampling:
```rust
pub fn select_arm(&mut self) -> &BanditArm {
self.total_pulls += 1;
let epsilon = (0.1 / (1.0 + self.total_pulls as f64 / 100.0)).max(0.02);
if rng_f64() < epsilon {
let idx = rng_usize(self.arms.len());
return &self.arms[idx];
}
let samples: Vec<f64> = self.arms.iter().map(BanditArm::sample).collect();
let best_idx = samples
.iter()
.enumerate()
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
.map_or(0, |(i, _)| i);
&self.arms[best_idx]
}
```
A design decision worth noting: we implemented Beta sampling from scratch using Marsaglia and Tsang's method for Gamma variates, rather than pulling in a statistics crate. The entire implementation is about 40 lines:
```rust
fn beta_sample(alpha: f64, beta: f64) -> f64 {
let x = gamma_sample(alpha);
let y = gamma_sample(beta);
if x + y == 0.0 { return 0.5; }
x / (x + y)
}
fn gamma_sample(shape: f64) -> f64 {
if shape < 1.0 {
let u = rng_f64().max(1e-10);
return gamma_sample(shape + 1.0) * u.powf(1.0 / shape);
}
let d = shape - 1.0 / 3.0;
let c = 1.0 / (9.0_f64 * d).sqrt();
loop {
let x = standard_normal();
let v = (1.0 + c * x).powi(3);
if v <= 0.0 { continue; }
let u = rng_f64().max(1e-10);
if u < 1.0 - 0.0331 * x.powi(4)
|| u.ln() < 0.5 * x * x + d * (1.0 - v + v.ln())
{
return d * v;
}
}
}
```
The bandit state (`alpha`, `beta` per arm) is serializable via serde — it persists across tool calls within a session but resets between sessions, since file characteristics change as the developer edits code.
## Shell pattern compression: OnceLock + modular dispatch
lean-ctx compresses shell output from 95+ command patterns (git, cargo, npm, docker, kubectl, etc.). The architecture uses a macro for zero-cost lazy regex initialization:
```rust
macro_rules! static_regex {
($pattern:expr) => {{
static RE: std::sync::OnceLock<regex::Regex> = std::sync::OnceLock::new();
RE.get_or_init(|| {
regex::Regex::new($pattern)
.expect(concat!("BUG: invalid static regex: ", $pattern))
})
}};
}
fn compiling_re() -> &'static regex::Regex {
static_regex!(r"Compiling (\S+) v(\S+)")
}
fn error_re() -> &'static regex::Regex {
static_regex!(r"error\[E(\d+)\]: (.+)")
}
```
Each tool family (cargo, git, npm, docker, etc.) gets its own module with a `compress` function. The dispatcher in `patterns/mod.rs` tries user-defined filters first, then routes to the specific pattern module based on command prefix. This makes adding a new pattern family a matter of adding a single file — no changes to the core dispatch logic.
For `cargo build`, the compressor extracts only errors, warnings, and the final status, discarding the hundreds of "Compiling foo v1.2.3" lines that an LLM doesn't need. A successful build that produced 200 lines of output gets compressed to something like `OK 42 crates in 12.3s` — from roughly 800 tokens to 15.
## Cache invalidation: MD5 for content, mtime for staleness
The session cache uses two signals to decide whether cached content is still valid:
```rust
pub fn is_cache_entry_stale(path: &str, cached_mtime: Option<SystemTime>) -> bool {
let current = file_mtime(path);
match (cached_mtime, current) {
(_, None) => false, // can't stat file — assume not stale
(None, Some(_)) => true, // no cached mtime — must re-read
(Some(cached), Some(current)) => current > cached,
}
}
```
When a file is first read, lean-ctx stores both an MD5 hash of the content and the filesystem mtime. On subsequent reads:
- If `mtime` hasn't changed, the cache is valid — return a compressed stub (~13 tokens instead of ~2,000).
- If `mtime` is newer, invalidate and re-read from disk. Even if the hash happens to match (the user saved without changes), we re-read to be safe.
- For `mode=full` reads, content hash comparison catches the case where mtime changed but content didn't.
This dual-signal approach was motivated by real user feedback: without mtime validation, LLMs would sometimes receive stale cached content after a file edit and get confused, spending more tokens trying to work around the stale data than the cache saved in the first place.
## Platform-specific output decoding
Shell output on Windows doesn't always arrive as UTF-8. lean-ctx handles this by falling back to the Windows Active Code Page via FFI — without pulling in a dedicated encoding crate:
```rust
pub fn decode_output(bytes: &[u8]) -> String {
match String::from_utf8(bytes.to_vec()) {
Ok(s) => s,
Err(_) => {
#[cfg(windows)]
{ decode_windows_output(bytes) }
#[cfg(not(windows))]
{ String::from_utf8_lossy(bytes).into_owned() }
}
}
}
#[cfg(windows)]
fn decode_windows_output(bytes: &[u8]) -> String {
extern "system" {
fn GetACP() -> u32;
fn MultiByteToWideChar(
cp: u32, flags: u32,
src: *const u8, srclen: i32,
dst: *mut u16, dstlen: i32,
) -> i32;
}
// ... two-pass: measure, allocate, convert
}
```
On Unix, container detection checks for `/.dockerenv` and `/proc/1/cgroup` to adjust behavior when running inside Docker or LXC — relevant because lean-ctx's shell hook needs to know whether it's intercepting commands in an isolated environment.
## Feature flags: one crate, many binaries
lean-ctx uses Cargo features extensively to keep the binary modular:
```toml
[features]
default = ["tree-sitter", "embeddings", "http-server"]
neural = ["dep:rten", "dep:rten-tensor"]
embeddings = ["dep:rten", "dep:rten-tensor"]
http-server = ["dep:axum", "dep:tower-http", "dep:reqwest"]
cloud-server = ["http-server", "dep:deadpool-postgres", ...]
tree-sitter = ["dep:tree-sitter", "dep:tree-sitter-rust", ...]
```
Building without `tree-sitter` drops 18 grammar crates and produces a significantly smaller binary — useful for constrained environments like Raspberry Pi or CI containers. The `cloud-server` feature pulls in Postgres, JWT, and SMTP dependencies that are only needed for the hosted API at leanctx.com.
## The MCP server: rmcp + async state
lean-ctx implements the Model Context Protocol using the `rmcp` crate. The server is a struct holding shared state behind `Arc<RwLock<...>>`:
```rust
#[derive(Clone)]
pub struct LeanCtxServer {
pub cache: SharedCache,
pub session: Arc<RwLock<SessionState>>,
pub tool_calls: Arc<RwLock<Vec<ToolCallRecord>>>,
pub call_count: Arc<AtomicUsize>,
pub loop_detector: Arc<RwLock<LoopDetector>>,
pub ledger: Arc<RwLock<ContextLedger>>,
// ...
}
```
Each tool call flows through `ServerHandler::call_tool` — an async method that routes to specific handlers (read, shell, search, etc.), applies post-processing (compression, archiving, throttle warnings), and tracks metrics. A meta-tool `ctx` allows the AI to call any lean-ctx tool by name, so the agent can write `ctx(tool="read", path="src/main.rs")` instead of `ctx_read(path="src/main.rs")` — reducing the number of tool definitions the model needs to track.
## Entropy: measuring information density with BPE tokens
lean-ctx has an `entropy` read mode that filters lines based on their information density. An interesting detail: we compute Shannon entropy twice — once over characters (standard) and once over BPE token IDs from `tiktoken-rs`:
```rust
pub fn shannon_entropy(text: &str) -> f64 {
let mut freq: HashMap<char, usize> = HashMap::new();
let total = text.chars().count();
for c in text.chars() {
*freq.entry(c).or_default() += 1;
}
freq.values().fold(0.0_f64, |acc, &count| {
let p = count as f64 / total as f64;
acc - p * p.log2()
})
}
pub fn token_entropy(text: &str) -> f64 {
let tokens = encode_tokens(text); // tiktoken o200k_base
let mut freq: HashMap<u32, usize> = HashMap::new();
for &t in &tokens {
*freq.entry(t).or_default() += 1;
}
// ... same formula over token IDs
}
```
Character entropy tells us how varied the text is syntactically. Token entropy tells us how varied it is from the LLM's perspective — a line that looks complex to humans might tokenize into common patterns, and vice versa. Using both signals together gives better filtering than either alone.
## Numbers
lean-ctx currently has 900+ GitHub stars and 32,000+ installs across npm, crates.io, and GitHub Releases. It supports 46 AI coding tools through one-command setup (`lean-ctx init --agent cursor`).
The codebase is roughly 35,000 lines of Rust, plus integration tests and benchmarks. The default binary (with tree-sitter and embeddings) compiles to about 15MB on x86_64.
---
*This post was written with AI assistance and reviewed by the author. lean-ctx is Apache-2.0 licensed.*
*Yves Gugger — [github.com/yvgude](https://github.com/yvgude)*
+54
View File
@@ -0,0 +1,54 @@
# GitLab CI — Website deployment only
# All Rust CI (test, clippy, fmt, benchmarks, coverage) runs on GitHub Actions.
# This pipeline is only active on the `deploy` branch for website builds.
workflow:
rules:
- if: $CI_COMMIT_BRANCH == "deploy"
- when: never
stages:
- build
- deploy
website_build:
stage: build
image: node:22.12.0
variables:
NPM_CONFIG_CACHE: "$CI_PROJECT_DIR/website/.npm"
cache:
key: "website-${CI_COMMIT_REF_SLUG}"
paths:
- website/.npm/
script:
- cd website
- npm ci
- npm run build
deploy_website:
stage: deploy
image: docker:24
needs: []
variables:
CONTAINER_NAME: leanctx-web
IMAGE_NAME: lean-ctx-web
DOCKER_HOST: unix:///var/run/docker.sock
script:
- echo "Building website image from source (multi-stage Dockerfile)..."
- test "$(git ls-files website/dist | wc -l)" -eq 0
- test "$(git ls-files website/node_modules | wc -l)" -eq 0
- cd website
- docker build -t ${IMAGE_NAME}:latest .
- echo "Stopping old container..."
- docker stop ${CONTAINER_NAME} || true
- docker rm ${CONTAINER_NAME} || true
- echo "Starting new container..."
- docker run -d
--name ${CONTAINER_NAME}
--network coolify
--restart unless-stopped
${IMAGE_NAME}:latest
- echo "Verifying..."
- sleep 2
- docker ps --filter name=${CONTAINER_NAME} --format "{{.Status}}"
- echo "Deploy complete!"
+8
View File
@@ -0,0 +1,8 @@
__pycache__/
*.py[cod]
*.egg-info/
.eggs/
build/
dist/
.pytest_cache/
.venv/
+123
View File
@@ -0,0 +1,123 @@
# lean-ctx-client (Python SDK)
Thin, **dependency-free** Python client for the lean-ctx HTTP `/v1` contract.
Standard library only (`urllib`) — installs and runs anywhere, no transitive
dependencies. It speaks the wire protocol only; it never links the engine or
re-implements compression. Mirrors the TypeScript and Rust
`lean-ctx-client` SDKs.
## Install
```bash
pip install lean-ctx-client
# from this repo:
pip install ./clients/python
```
## Usage
```python
from leanctx import LeanCtxClient, run_conformance
client = LeanCtxClient("http://127.0.0.1:8080")
# Discovery
caps = client.capabilities() # GET /v1/capabilities
api = client.openapi() # GET /v1/openapi.json
# Tools
listing = client.list_tools(limit=10)
text = client.call_tool_text("ctx_read", {"path": "README.md"})
# Live events (SSE)
for event in client.subscribe_events():
print(event["kind"], event["payload"])
```
## Methods
| Method | Endpoint |
|--------|----------|
| `health()` | `GET /health` |
| `manifest()` | `GET /v1/manifest` |
| `capabilities()` | `GET /v1/capabilities` |
| `openapi()` | `GET /v1/openapi.json` |
| `list_tools(offset=, limit=)` | `GET /v1/tools` |
| `call_tool(name, arguments, ...)` | `POST /v1/tools/call` |
| `call_tool_text(name, arguments, ...)` | `POST /v1/tools/call` + text extraction |
| `subscribe_events(...)` | `GET /v1/events` (SSE) |
| `context_summary(...)` | `GET /v1/context/summary` |
| `search_events(q, ...)` | `GET /v1/events/search` |
| `event_lineage(event_id, ...)` | `GET /v1/events/lineage` |
| `metrics()` | `GET /v1/metrics` |
## Shared conformance kit
`run_conformance(client)` runs the language-agnostic SDK conformance checks
against a live server and returns a scorecard. It mirrors the TypeScript SDK's
`runConformance` and the server-side `lean-ctx conformance`, keeping all clients
in lockstep on the same contract.
```python
card = run_conformance(client)
assert card.all_passed, [c for c in card.checks if not c.passed]
```
The kit covers **every** `/v1` route (`COVERED_ROUTES`): its `route_coverage`
check fails when the server advertises a route this SDK does not cover, and
`engine_compat` fails when the server speaks an `http_mcp` contract version
outside `SUPPORTED_HTTP_CONTRACT_VERSIONS`. The published matrix lives at
`docs/reference/sdk-conformance-matrix.md` (regenerated by
`scripts/sdk-conformance.sh` against a real server in the `sdk-conformance`
CI job).
### SemVer coupling
The SDK's major version follows the engine's `http_mcp` contract major
(CONTRACTS.md § Versioning rules): a `v2` contract ships as SDK `2.x`, and
`1.x` keeps speaking `v1`.
## Framework adapters
Expose the lean-ctx tool surface to popular agent frameworks. Each framework is
an **optional** dependency, imported lazily — installing `lean-ctx-client` pulls in none
of them. The OpenAI adapter is a pure transformation and needs no extra package.
```python
from leanctx import LeanCtxClient
from leanctx.adapters import (
to_openai_tools, run_openai_tool_call, # no extra dep
to_langchain_tools, # pip install "lean-ctx-client[langchain]"
to_llamaindex_tools, # pip install "lean-ctx-client[llamaindex]"
to_crewai_tools, # pip install "lean-ctx-client[crewai]"
)
client = LeanCtxClient("http://127.0.0.1:8080")
# OpenAI function calling
tools = to_openai_tools(client)
# ... pass tools to client.chat.completions.create(...), then:
text = run_openai_tool_call(client, tool_call)
# LangChain / LlamaIndex / CrewAI
lc_tools = to_langchain_tools(client)
```
## Errors
- `LeanCtxConfigError` — invalid arguments / configuration (no I/O performed).
- `LeanCtxTransportError` — request never produced a response (network/DNS/TLS).
- `LeanCtxHTTPError` — non-2xx response, with `status`, `method`, `url`,
`error_code`, and parsed `body`.
## Non-goals
- No engine linkage and no re-implemented compression/indexing logic.
- Stability over surface: only the documented `/v1` contract is exposed.
## Development
```bash
cd clients/python
python -m pytest
```
+40
View File
@@ -0,0 +1,40 @@
"""lean-ctx Python SDK.
A thin, dependency-free client for the lean-ctx HTTP ``/v1`` contract. Mirrors
the TypeScript (`lean-ctx-client`) and Rust (`lean-ctx-client`) SDKs.
"""
from __future__ import annotations
from .client import LeanCtxClient
from .conformance import (
COVERED_ROUTES,
SUPPORTED_HTTP_CONTRACT_VERSIONS,
ConformanceCheck,
ConformanceScorecard,
run_conformance,
)
from .errors import (
LeanCtxConfigError,
LeanCtxError,
LeanCtxHTTPError,
LeanCtxTransportError,
)
from .tool_text import tool_result_to_text
__version__ = "0.1.0"
__all__ = [
"LeanCtxClient",
"LeanCtxError",
"LeanCtxConfigError",
"LeanCtxTransportError",
"LeanCtxHTTPError",
"tool_result_to_text",
"run_conformance",
"ConformanceCheck",
"ConformanceScorecard",
"COVERED_ROUTES",
"SUPPORTED_HTTP_CONTRACT_VERSIONS",
"__version__",
]
@@ -0,0 +1,32 @@
"""Framework adapters for lean-ctx tools (EPIC 12.6).
Thin, optional integrations that expose the lean-ctx tool surface to popular
agent frameworks. Each framework is an *optional* dependency, imported lazily —
installing the SDK pulls in none of them. The OpenAI adapter is a pure
transformation and needs no extra package.
from leanctx import LeanCtxClient
from leanctx.adapters import to_openai_tools, to_langchain_tools
client = LeanCtxClient("http://127.0.0.1:8080")
tools = to_openai_tools(client)
"""
from __future__ import annotations
from ._common import ToolSpec, coerce_arguments, normalized_tool_specs
from .crewai import to_crewai_tools
from .langchain import to_langchain_tools
from .llamaindex import to_llamaindex_tools
from .openai import run_openai_tool_call, to_openai_tools
__all__ = [
"ToolSpec",
"normalized_tool_specs",
"coerce_arguments",
"to_openai_tools",
"run_openai_tool_call",
"to_langchain_tools",
"to_llamaindex_tools",
"to_crewai_tools",
]
@@ -0,0 +1,83 @@
"""Shared helpers for framework adapters (EPIC 12.6).
All adapters normalize lean-ctx tools into a common [`ToolSpec`] and reuse the
same SDK call path (`call_tool_text`), so every framework integration behaves
identically and stays correct as the tool surface evolves.
"""
from __future__ import annotations
import json
from dataclasses import dataclass
from typing import Any, Callable, Dict, List
from ..client import LeanCtxClient
@dataclass
class ToolSpec:
"""A framework-neutral description of one lean-ctx tool."""
name: str
description: str
parameters: Dict[str, Any] # JSON Schema for the arguments object
def _default_schema() -> Dict[str, Any]:
return {"type": "object", "properties": {}}
def normalized_tool_specs(client: LeanCtxClient, *, limit: int = 500) -> List[ToolSpec]:
"""Fetch and normalize the server's tool surface into [`ToolSpec`]s."""
listing = client.list_tools(limit=limit)
tools = listing.get("tools", []) if isinstance(listing, dict) else []
specs: List[ToolSpec] = []
for entry in tools:
if not isinstance(entry, dict):
continue
name = entry.get("name")
if not isinstance(name, str) or not name:
continue
description = entry.get("description")
if not isinstance(description, str):
description = ""
schema = (
entry.get("input_schema")
or entry.get("inputSchema")
or entry.get("parameters")
or _default_schema()
)
if not isinstance(schema, dict):
schema = _default_schema()
specs.append(ToolSpec(name=name, description=description, parameters=schema))
return specs
def coerce_arguments(raw: Any) -> Dict[str, Any]:
"""Coerce tool-call arguments (JSON string or dict) into a dict."""
if raw is None:
return {}
if isinstance(raw, dict):
return raw
if isinstance(raw, str):
text = raw.strip()
if not text:
return {}
parsed = json.loads(text)
return parsed if isinstance(parsed, dict) else {}
return {}
def make_json_runner(client: LeanCtxClient, name: str) -> Callable[[str], str]:
"""A `(arguments: str) -> str` runner used by string-input frameworks.
The single JSON-string argument is the most portable shape across framework
versions and avoids brittle per-tool signature synthesis.
"""
def run(arguments: str = "") -> str:
return client.call_tool_text(name, coerce_arguments(arguments))
run.__name__ = name
run.__doc__ = f"Invoke the lean-ctx tool '{name}' with a JSON object string."
return run
+53
View File
@@ -0,0 +1,53 @@
"""CrewAI adapter (EPIC 12.6).
Exposes lean-ctx tools as CrewAI `BaseTool`s. The framework is an optional
dependency, imported lazily.
"""
from __future__ import annotations
from typing import Any, List
from ..client import LeanCtxClient
from ._common import coerce_arguments, normalized_tool_specs
def to_crewai_tools(client: LeanCtxClient) -> List[Any]:
"""Return lean-ctx tools as CrewAI `BaseTool` instances.
Each tool takes a single ``arguments`` field: a JSON object string of the
tool's arguments. This is portable across CrewAI versions and avoids
synthesizing a distinct pydantic schema per tool.
"""
try:
from crewai.tools import BaseTool
from pydantic import BaseModel, Field
except ImportError as exc: # pragma: no cover - exercised only without dep
raise ImportError(
"CrewAI is required for this adapter: pip install crewai"
) from exc
class _ArgsSchema(BaseModel):
arguments: str = Field(
default="",
description="JSON object string of the tool's arguments.",
)
class _LeanCtxTool(BaseTool):
args_schema: type[BaseModel] = _ArgsSchema
def __init__(self, tool_name: str, tool_description: str) -> None:
super().__init__(name=tool_name, description=tool_description)
self._tool_name = tool_name
def _run(self, arguments: str = "") -> str:
return client.call_tool_text(self._tool_name, coerce_arguments(arguments))
tools: List[Any] = []
for spec in normalized_tool_specs(client):
description = (
f"{spec.description} "
"Input: a JSON object string matching this tool's argument schema."
).strip()
tools.append(_LeanCtxTool(spec.name, description))
return tools
@@ -0,0 +1,36 @@
"""LangChain adapter (EPIC 12.6).
Exposes lean-ctx tools as LangChain `Tool`s. The framework is an optional
dependency, imported lazily so the SDK has no hard coupling.
"""
from __future__ import annotations
from typing import Any, List
from ..client import LeanCtxClient
from ._common import make_json_runner, normalized_tool_specs
def to_langchain_tools(client: LeanCtxClient) -> List[Any]:
"""Return lean-ctx tools as `langchain_core.tools.Tool` instances.
Each tool accepts a JSON object string of arguments — the most portable
shape across LangChain versions.
"""
try:
from langchain_core.tools import Tool
except ImportError as exc: # pragma: no cover - exercised only without dep
raise ImportError(
"LangChain is required for this adapter: pip install langchain-core"
) from exc
tools: List[Any] = []
for spec in normalized_tool_specs(client):
runner = make_json_runner(client, spec.name)
description = (
f"{spec.description} "
"Input: a JSON object string matching this tool's argument schema."
).strip()
tools.append(Tool(name=spec.name, description=description, func=runner))
return tools
@@ -0,0 +1,36 @@
"""LlamaIndex adapter (EPIC 12.6).
Exposes lean-ctx tools as LlamaIndex `FunctionTool`s. The framework is an
optional dependency, imported lazily.
"""
from __future__ import annotations
from typing import Any, List
from ..client import LeanCtxClient
from ._common import make_json_runner, normalized_tool_specs
def to_llamaindex_tools(client: LeanCtxClient) -> List[Any]:
"""Return lean-ctx tools as `llama_index.core.tools.FunctionTool` instances."""
try:
from llama_index.core.tools import FunctionTool
except ImportError as exc: # pragma: no cover - exercised only without dep
raise ImportError(
"LlamaIndex is required for this adapter: pip install llama-index-core"
) from exc
tools: List[Any] = []
for spec in normalized_tool_specs(client):
runner = make_json_runner(client, spec.name)
description = (
f"{spec.description} "
"Input: a JSON object string matching this tool's argument schema."
).strip()
tools.append(
FunctionTool.from_defaults(
fn=runner, name=spec.name, description=description
)
)
return tools
+46
View File
@@ -0,0 +1,46 @@
"""OpenAI function-calling adapter (EPIC 12.6).
Pure transformation — no `openai` package required. Turns the lean-ctx tool
surface into OpenAI ``tools=[...]`` specs and executes the tool calls the model
returns. Works with both the Chat Completions and Responses tool-call shapes.
"""
from __future__ import annotations
from typing import Any, Dict, List
from ..client import LeanCtxClient
from ._common import coerce_arguments, normalized_tool_specs
def to_openai_tools(client: LeanCtxClient) -> List[Dict[str, Any]]:
"""Return lean-ctx tools as OpenAI function-tool specs."""
return [
{
"type": "function",
"function": {
"name": spec.name,
"description": spec.description,
"parameters": spec.parameters,
},
}
for spec in normalized_tool_specs(client)
]
def run_openai_tool_call(client: LeanCtxClient, tool_call: Any) -> str:
"""Execute one OpenAI tool call and return the tool's text result.
Accepts either the dict shape (``{"function": {"name", "arguments"}}``) or
an SDK object exposing ``.function.name`` / ``.function.arguments``.
"""
function = tool_call.get("function") if isinstance(tool_call, dict) else getattr(tool_call, "function", None)
if function is None:
raise ValueError("tool_call has no 'function'")
name = function.get("name") if isinstance(function, dict) else getattr(function, "name", None)
if not name:
raise ValueError("tool_call.function has no 'name'")
raw_args = (
function.get("arguments") if isinstance(function, dict) else getattr(function, "arguments", None)
)
return client.call_tool_text(name, coerce_arguments(raw_args))
+351
View File
@@ -0,0 +1,351 @@
"""Thin, dependency-free HTTP client for the lean-ctx ``/v1`` contract.
Uses only the Python standard library (``urllib``) so it installs and runs
anywhere with no transitive dependencies. It speaks the wire protocol only — it
never links the engine or re-implements compression — so it stays stable as
lean-ctx evolves. Mirrors the TypeScript (`lean-ctx-client`) and Rust
(`lean-ctx-client`) SDKs.
"""
from __future__ import annotations
import json
import urllib.error
import urllib.parse
import urllib.request
from typing import Any, Dict, Iterator, Optional
from .errors import LeanCtxConfigError, LeanCtxHTTPError, LeanCtxTransportError
from .tool_text import tool_result_to_text
class LeanCtxClient:
"""A blocking client for a running lean-ctx HTTP server."""
def __init__(
self,
base_url: str,
*,
bearer_token: Optional[str] = None,
workspace_id: Optional[str] = None,
channel_id: Optional[str] = None,
timeout: float = 30.0,
) -> None:
trimmed = (base_url or "").strip()
if not trimmed:
raise LeanCtxConfigError("base_url is required")
self.base_url = trimmed[:-1] if trimmed.endswith("/") else trimmed
self._bearer_token = (bearer_token or "").strip() or None
self._workspace_id = (workspace_id or "").strip() or None
self._channel_id = (channel_id or "").strip() or None
self._timeout = timeout
# -- discovery ---------------------------------------------------------
def health(self) -> str:
return self._request_text("GET", "/health")
def manifest(self) -> Any:
return self._get_json("/v1/manifest")
def capabilities(self) -> Dict[str, Any]:
return self._get_json("/v1/capabilities")
def openapi(self) -> Dict[str, Any]:
return self._get_json("/v1/openapi.json")
# -- tools -------------------------------------------------------------
def list_tools(
self, *, offset: Optional[int] = None, limit: Optional[int] = None
) -> Dict[str, Any]:
query: Dict[str, str] = {}
if offset is not None:
query["offset"] = str(offset)
if limit is not None:
query["limit"] = str(limit)
return self._get_json("/v1/tools", query)
def call_tool(
self,
name: str,
arguments: Optional[Dict[str, Any]] = None,
*,
workspace_id: Optional[str] = None,
channel_id: Optional[str] = None,
) -> Any:
"""Call a tool and return its raw ``result``."""
if not name:
raise LeanCtxConfigError("tool name is required")
if arguments is not None and not isinstance(arguments, dict):
raise LeanCtxConfigError("arguments must be a dict (JSON object)")
body: Dict[str, Any] = {"name": name}
if arguments is not None:
body["arguments"] = arguments
ws = (workspace_id or "").strip() or self._workspace_id
ch = (channel_id or "").strip() or self._channel_id
if ws:
body["workspaceId"] = ws
if ch:
body["channelId"] = ch
payload = self._request_json(
"POST", "/v1/tools/call", body=body, workspace_id=ws
)
if not isinstance(payload, dict):
raise LeanCtxConfigError("call_tool: unexpected response shape")
return payload.get("result")
def call_tool_text(
self,
name: str,
arguments: Optional[Dict[str, Any]] = None,
**ctx: Any,
) -> str:
"""Call a tool and flatten its result into text."""
return tool_result_to_text(self.call_tool(name, arguments, **ctx))
# -- context views -------------------------------------------------------
def context_summary(
self,
*,
workspace_id: Optional[str] = None,
channel_id: Optional[str] = None,
limit: Optional[int] = None,
) -> Dict[str, Any]:
"""Materialized workspace/channel summary (``GET /v1/context/summary``)."""
query: Dict[str, str] = {}
ws = (workspace_id or "").strip() or self._workspace_id
ch = (channel_id or "").strip() or self._channel_id
if ws:
query["workspaceId"] = ws
if ch:
query["channelId"] = ch
if limit is not None:
query["limit"] = str(limit)
return self._get_json("/v1/context/summary", query)
def search_events(
self,
q: str,
*,
workspace_id: Optional[str] = None,
channel_id: Optional[str] = None,
limit: Optional[int] = None,
) -> Dict[str, Any]:
"""Full-text search over event payloads (``GET /v1/events/search``)."""
if not q:
raise LeanCtxConfigError("search query is required")
query: Dict[str, str] = {"q": q}
ws = (workspace_id or "").strip() or self._workspace_id
ch = (channel_id or "").strip() or self._channel_id
if ws:
query["workspaceId"] = ws
if ch:
query["channelId"] = ch
if limit is not None:
query["limit"] = str(limit)
return self._get_json("/v1/events/search", query)
def event_lineage(
self,
event_id: int,
*,
depth: Optional[int] = None,
workspace_id: Optional[str] = None,
) -> Dict[str, Any]:
"""Causal lineage chain for an event (``GET /v1/events/lineage``)."""
query: Dict[str, str] = {"id": str(event_id)}
if depth is not None:
query["depth"] = str(depth)
ws = (workspace_id or "").strip() or self._workspace_id
if ws:
query["workspaceId"] = ws
return self._get_json("/v1/events/lineage", query)
def metrics(self) -> Dict[str, Any]:
"""JSON metrics snapshot (``GET /v1/metrics``)."""
return self._get_json("/v1/metrics")
# -- events (SSE) ------------------------------------------------------
def subscribe_events(
self,
*,
workspace_id: Optional[str] = None,
channel_id: Optional[str] = None,
since: Optional[int] = None,
limit: Optional[int] = None,
) -> Iterator[Dict[str, Any]]:
"""Yield ``ContextEventV1`` dicts from the SSE stream until the server closes it."""
ws = (workspace_id or "").strip() or self._workspace_id
ch = (channel_id or "").strip() or self._channel_id
query: Dict[str, str] = {}
if ws:
query["workspaceId"] = ws
if ch:
query["channelId"] = ch
if since is not None:
query["since"] = str(since)
if limit is not None:
query["limit"] = str(limit)
req = self._build_request(
"GET", "/v1/events", query, accept="text/event-stream", workspace_id=ws
)
try:
resp = urllib.request.urlopen(req, timeout=self._timeout)
except urllib.error.HTTPError as exc:
raise self._http_error_from(exc, "GET", "/v1/events") from exc
except urllib.error.URLError as exc:
raise LeanCtxTransportError(str(exc.reason)) from exc
with resp:
buf = ""
for raw in resp:
buf += raw.decode("utf-8", "replace")
while "\n\n" in buf:
chunk, buf = buf.split("\n\n", 1)
data = _parse_sse_data(chunk)
if data is None:
continue
try:
event = json.loads(data)
except json.JSONDecodeError:
continue
if isinstance(event, dict):
yield event
def events_probe(self) -> str:
"""Open ``GET /v1/events`` and return its ``Content-Type`` header.
Returns as soon as response headers arrive (the body is never read),
so it cannot block on an idle stream. Used by the conformance kit to
prove the SSE endpoint exists and speaks ``text/event-stream``.
"""
req = self._build_request(
"GET", "/v1/events", {"limit": "1"}, accept="text/event-stream"
)
try:
resp = urllib.request.urlopen(req, timeout=self._timeout)
except urllib.error.HTTPError as exc:
raise self._http_error_from(exc, "GET", "/v1/events") from exc
except urllib.error.URLError as exc:
raise LeanCtxTransportError(str(exc.reason)) from exc
with resp:
return resp.headers.get("content-type", "") or ""
# -- internals ---------------------------------------------------------
def _get_json(self, path: str, query: Optional[Dict[str, str]] = None) -> Any:
return self._request_json("GET", path, query=query)
def _request_text(self, method: str, path: str) -> str:
req = self._build_request(method, path, accept="text/plain")
return self._send(req, method, path).decode("utf-8", "replace")
def _request_json(
self,
method: str,
path: str,
*,
query: Optional[Dict[str, str]] = None,
body: Optional[Dict[str, Any]] = None,
workspace_id: Optional[str] = None,
) -> Any:
req = self._build_request(
method,
path,
query,
accept="application/json",
body=body,
workspace_id=workspace_id,
)
raw = self._send(req, method, path)
if not raw:
return None
return json.loads(raw.decode("utf-8", "replace"))
def _build_request(
self,
method: str,
path: str,
query: Optional[Dict[str, str]] = None,
*,
accept: str,
body: Optional[Dict[str, Any]] = None,
workspace_id: Optional[str] = None,
) -> urllib.request.Request:
url = self.base_url + path
if query:
url += "?" + urllib.parse.urlencode(query)
data = None
headers = {"Accept": accept}
if body is not None:
data = json.dumps(body).encode("utf-8")
headers["Content-Type"] = "application/json"
if self._bearer_token:
headers["Authorization"] = f"Bearer {self._bearer_token}"
if workspace_id:
headers["x-leanctx-workspace"] = workspace_id
return urllib.request.Request(url, data=data, headers=headers, method=method)
def _send(self, req: urllib.request.Request, method: str, path: str) -> bytes:
try:
with urllib.request.urlopen(req, timeout=self._timeout) as resp:
return resp.read()
except urllib.error.HTTPError as exc:
raise self._http_error_from(exc, method, path) from exc
except urllib.error.URLError as exc:
raise LeanCtxTransportError(str(exc.reason)) from exc
def _http_error_from(
self, exc: urllib.error.HTTPError, method: str, path: str
) -> LeanCtxHTTPError:
url = self.base_url + path
message = f"HTTP {exc.code} {method} {url}"
error_code = None
body: Any = None
try:
raw = exc.read()
except Exception: # pragma: no cover - defensive
raw = b""
content_type = exc.headers.get("content-type", "") if exc.headers else ""
if raw:
if "application/json" in content_type:
try:
body = json.loads(raw.decode("utf-8", "replace"))
if isinstance(body, dict):
if isinstance(body.get("error"), str) and body["error"].strip():
message = body["error"]
if isinstance(body.get("error_code"), str):
error_code = body["error_code"].strip() or None
except json.JSONDecodeError:
body = raw.decode("utf-8", "replace")
else:
text = raw.decode("utf-8", "replace")
body = text
if text.strip():
message = text.strip()
return LeanCtxHTTPError(
status=exc.code,
method=method,
url=url,
message=message,
error_code=error_code,
body=body,
)
def _parse_sse_data(chunk: str) -> Optional[str]:
"""Join the ``data:`` lines of one SSE frame; ignore id/event/comments."""
data_lines = []
for line in chunk.split("\n"):
line = line.rstrip("\r")
if not line or line.startswith(":"):
continue
if line.startswith("data:"):
data_lines.append(line[5:].lstrip())
return "\n".join(data_lines) if data_lines else None
+217
View File
@@ -0,0 +1,217 @@
"""Shared SDK conformance kit (EPIC 12.4/12.5, industrialized in GL #395).
A client-side check that proves the Python SDK + a live server interoperate over
the **entire** frozen ``/v1`` contract. It is the exact mirror of the TypeScript
SDK's ``runConformance`` and the Rust client's ``run_conformance``, so every
first-party SDK proves the same contract and they stay in lockstep.
Two checks make this a drift gate (GL #395):
* ``route_coverage`` — every path the server's OpenAPI document advertises must
be covered by an SDK method (``COVERED_ROUTES``). A new server route without
SDK support fails conformance in the next CI run.
* ``engine_compat`` — the server's ``http_mcp`` contract version must be one
this SDK release supports (``SUPPORTED_HTTP_CONTRACT_VERSIONS``).
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List
from .client import LeanCtxClient
from .errors import LeanCtxHTTPError
#: ``METHOD path`` → client method name. The conformance kit fails if the live
#: server's OpenAPI document lists a route that is missing here.
COVERED_ROUTES: Dict[str, str] = {
"GET /health": "health",
"GET /v1/manifest": "manifest",
"GET /v1/capabilities": "capabilities",
"GET /v1/openapi.json": "openapi",
"GET /v1/tools": "list_tools",
"POST /v1/tools/call": "call_tool",
"GET /v1/events": "subscribe_events",
"GET /v1/context/summary": "context_summary",
"GET /v1/events/search": "search_events",
"GET /v1/events/lineage": "event_lineage",
"GET /v1/metrics": "metrics",
}
#: ``http_mcp`` contract versions this SDK release speaks (SemVer coupling:
#: the SDK major follows the engine contract major).
SUPPORTED_HTTP_CONTRACT_VERSIONS = (1,)
@dataclass
class ConformanceCheck:
name: str
passed: bool
detail: str = ""
@dataclass
class ConformanceScorecard:
checks: List[ConformanceCheck] = field(default_factory=list)
@property
def passed(self) -> int:
return sum(1 for c in self.checks if c.passed)
@property
def total(self) -> int:
return len(self.checks)
@property
def all_passed(self) -> bool:
return all(c.passed for c in self.checks)
def _add(card: ConformanceScorecard, name: str, probe: Callable[[], Any]) -> None:
"""Run one probe; contract/network failures become failed checks."""
try:
ok, detail = probe()
card.checks.append(ConformanceCheck(name, bool(ok), detail))
except Exception as exc: # noqa: BLE001 - capture as a failed check
card.checks.append(ConformanceCheck(name, False, str(exc)))
def run_conformance(client: LeanCtxClient) -> ConformanceScorecard:
"""Run the conformance kit against a live client.
Network/contract failures become failed checks rather than exceptions, so
the returned scorecard is always complete and comparable across SDKs.
"""
card = ConformanceScorecard()
def health() -> Any:
return isinstance(client.health(), str), ""
def manifest_shape() -> Any:
m = client.manifest()
return isinstance(m, dict) and bool(m), ""
def capabilities_shape() -> Any:
caps = client.capabilities()
server = caps.get("server", {}) if isinstance(caps, dict) else {}
ok = (
isinstance(caps, dict)
and isinstance(caps.get("contract_version"), int)
and isinstance(server, dict)
and bool(server.get("version"))
and isinstance(caps.get("plane"), str)
and isinstance(caps.get("transports"), list)
and isinstance(caps.get("features"), dict)
and isinstance(caps.get("contracts"), dict)
)
return ok, ""
def contract_status_map() -> Any:
# GL #394: stability per contract is part of the discovery document.
status = client.capabilities().get("contract_status")
ok = isinstance(status, dict) and status.get("http-mcp") in (
"frozen",
"stable",
)
return ok, "" if ok else f"contract_status={status!r}"
def engine_compat() -> Any:
contracts = client.capabilities().get("contracts", {})
version = contracts.get("leanctx.contract.http_mcp.contract_version")
ok = version in SUPPORTED_HTTP_CONTRACT_VERSIONS
return ok, "" if ok else f"server http_mcp contract v{version!r} unsupported"
def openapi_shape() -> Any:
doc = client.openapi()
version = doc.get("openapi", "") if isinstance(doc, dict) else ""
ok = (
isinstance(version, str)
and version.startswith("3.")
and isinstance(doc.get("paths"), dict)
)
return ok, ""
def route_coverage() -> Any:
# The drift gate: every advertised route needs an SDK method.
doc = client.openapi()
paths = doc.get("paths", {}) if isinstance(doc, dict) else {}
uncovered = [
f"{method.upper()} {path}"
for path, ops in paths.items()
if isinstance(ops, dict)
for method in ops
if f"{method.upper()} {path}" not in COVERED_ROUTES
]
return not uncovered, ", ".join(sorted(uncovered))
def tools_list() -> Any:
listing = client.list_tools(limit=1)
ok = (
isinstance(listing, dict)
and isinstance(listing.get("tools"), list)
and isinstance(listing.get("total"), int)
and listing["total"] >= 0
)
return ok, ""
def tool_call_error_contract() -> Any:
# Typed-error semantics: an unknown tool must produce a structured
# 4xx with a machine-readable error_code, not a 5xx or free text.
try:
client.call_tool("definitely_not_a_tool_conformance_probe")
except LeanCtxHTTPError as exc:
ok = 400 <= exc.status < 500 and bool(exc.error_code)
return ok, "" if ok else f"status={exc.status} error_code={exc.error_code!r}"
return False, "unknown tool call unexpectedly succeeded"
def events_stream() -> Any:
content_type = client.events_probe()
ok = content_type.startswith("text/event-stream")
return ok, "" if ok else f"content-type={content_type!r}"
def context_summary_shape() -> Any:
summary = client.context_summary(limit=1)
ok = (
isinstance(summary, dict)
and isinstance(summary.get("workspaceId"), str)
and isinstance(summary.get("totalEvents"), int)
and isinstance(summary.get("eventCountsByKind"), dict)
)
return ok, ""
def events_search_shape() -> Any:
res = client.search_events("conformance-probe", limit=1)
ok = (
isinstance(res, dict)
and isinstance(res.get("results"), list)
and isinstance(res.get("count"), int)
)
return ok, ""
def event_lineage_shape() -> Any:
res = client.event_lineage(1, depth=1)
ok = (
isinstance(res, dict)
and "eventId" in res
and isinstance(res.get("chain"), list)
)
return ok, ""
def metrics_shape() -> Any:
return isinstance(client.metrics(), dict), ""
_add(card, "health", health)
_add(card, "manifest_shape", manifest_shape)
_add(card, "capabilities_shape", capabilities_shape)
_add(card, "contract_status_map", contract_status_map)
_add(card, "engine_compat", engine_compat)
_add(card, "openapi_shape", openapi_shape)
_add(card, "route_coverage", route_coverage)
_add(card, "tools_list", tools_list)
_add(card, "tool_call_error_contract", tool_call_error_contract)
_add(card, "events_stream", events_stream)
_add(card, "context_summary_shape", context_summary_shape)
_add(card, "events_search_shape", events_search_shape)
_add(card, "event_lineage_shape", event_lineage_shape)
_add(card, "metrics_shape", metrics_shape)
return card
+47
View File
@@ -0,0 +1,47 @@
"""Error types for the lean-ctx Python client."""
from __future__ import annotations
from typing import Any, Optional
class LeanCtxError(Exception):
"""Base class for all lean-ctx client errors."""
class LeanCtxConfigError(LeanCtxError):
"""Raised for invalid client configuration or arguments (no I/O performed)."""
class LeanCtxTransportError(LeanCtxError):
"""Raised when the request never produced an HTTP response (network/DNS/TLS)."""
class LeanCtxHTTPError(LeanCtxError):
"""Raised for a non-2xx HTTP response from the server.
Carries enough structured detail (status, method, url, server error code and
body) for programmatic handling, mirroring the Rust/TS SDK error shape.
"""
def __init__(
self,
*,
status: int,
method: str,
url: str,
message: str,
error_code: Optional[str] = None,
body: Any = None,
) -> None:
super().__init__(message)
self.status = status
self.method = method
self.url = url
self.message = message
self.error_code = error_code
self.body = body
def __str__(self) -> str: # pragma: no cover - trivial
code = f" [{self.error_code}]" if self.error_code else ""
return f"HTTP {self.status} {self.method} {self.url}{code}: {self.message}"

Some files were not shown because too many files have changed in this diff Show More