#!/usr/bin/env bash set -euo pipefail # lean-ctx Benchmark v2.0 # Uses lean-ctx's built-in tiktoken tokenizer for exact measurements. # No char/4 approximations. SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" RESULTS_DIR="$SCRIPT_DIR/results" LEAN_CTX="${LEAN_CTX_BIN:-lean-ctx}" mkdir -p "$RESULTS_DIR" echo "═══════════════════════════════════════════════════" echo " lean-ctx Benchmark v2.0" echo " Project: $PROJECT_ROOT" echo " Binary: $($LEAN_CTX --version 2>/dev/null || echo 'unknown')" echo " Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "═══════════════════════════════════════════════════" echo "" # ═══════════════════════════════════════════════════════ # CATEGORY 1: CLI Output Filtering (vs. RTK) # Uses lean-ctx's built-in token counter for accuracy # ═══════════════════════════════════════════════════════ echo "╔═══ Category 1: CLI Output Filtering ═══╗" echo "" declare -a CLI_COMMANDS=( "git log --oneline -20" "git status" "git diff HEAD~3 --stat" "git branch -a" "git log --format='%h %s %an %ar' -30" "ls -la" "ls -laR rust/src/core/patterns/" "cargo test --lib -- --list 2>&1 | head -100" "cargo tree --depth 1" "git log --stat -5" "git shortlog -sn --all" "git remote -v" "cat Cargo.toml" "cat rust/Cargo.toml" "env | sort | head -40" "ps aux | head -30" "df -h" ) CLI_TOTAL_RAW=0 CLI_TOTAL_COMPRESSED=0 CLI_COUNT=0 for cmd in "${CLI_COMMANDS[@]}"; do # Get raw output RAW_OUTPUT=$(cd "$PROJECT_ROOT" && eval "$cmd" 2>/dev/null || true) if [ -z "$RAW_OUTPUT" ] || [ ${#RAW_OUTPUT} -lt 10 ]; then continue fi # Count tokens via lean-ctx's tokenizer (pipe to token count) RAW_TOKENS=$(echo "$RAW_OUTPUT" | "$LEAN_CTX" tokens count 2>/dev/null || echo "$RAW_OUTPUT" | wc -m | awk '{print int($1/3.5)}') # Get compressed output (suppress savings footer) COMPRESSED_OUTPUT=$(cd "$PROJECT_ROOT" && LEAN_CTX_SAVINGS_FOOTER=0 "$LEAN_CTX" -c "$cmd" 2>/dev/null || true) COMPRESSED_TOKENS=$(echo "$COMPRESSED_OUTPUT" | "$LEAN_CTX" tokens count 2>/dev/null || echo "$COMPRESSED_OUTPUT" | wc -m | awk '{print int($1/3.5)}') if [ "$RAW_TOKENS" -gt 10 ]; then CLI_TOTAL_RAW=$((CLI_TOTAL_RAW + RAW_TOKENS)) CLI_TOTAL_COMPRESSED=$((CLI_TOTAL_COMPRESSED + COMPRESSED_TOKENS)) CLI_COUNT=$((CLI_COUNT + 1)) SAVINGS_PCT=$(echo "scale=1; ($RAW_TOKENS - $COMPRESSED_TOKENS) * 100 / $RAW_TOKENS" | bc 2>/dev/null || echo "0") printf " [%2d] %-45s %6s → %6s (-%s%%)\n" "$CLI_COUNT" "$cmd" "$RAW_TOKENS" "$COMPRESSED_TOKENS" "$SAVINGS_PCT" fi done CLI_SAVINGS_PCT=$(echo "scale=1; ($CLI_TOTAL_RAW - $CLI_TOTAL_COMPRESSED) * 100 / $CLI_TOTAL_RAW" | bc 2>/dev/null || echo "0") echo "" echo " ── CLI Summary: ${CLI_COUNT} commands" echo " Total raw: ${CLI_TOTAL_RAW} tokens" echo " Total compressed: ${CLI_TOTAL_COMPRESSED} tokens" echo " Savings: ${CLI_SAVINGS_PCT}%" echo "" # ═══════════════════════════════════════════════════════ # CATEGORY 2: File Read Compression # Uses lean-ctx's built-in benchmark (tiktoken exact) # ═══════════════════════════════════════════════════════ echo "╔═══ Category 2: File Read Compression (tiktoken exact) ═══╗" echo "" # Run built-in benchmark with JSON output BENCHMARK_JSON=$(cd "$PROJECT_ROOT" && "$LEAN_CTX" benchmark run . --json 2>/dev/null) if [ -z "$BENCHMARK_JSON" ]; then echo " ERROR: lean-ctx benchmark returned empty output" exit 1 fi # Save raw benchmark JSON echo "$BENCHMARK_JSON" > "$RESULTS_DIR/benchmark-raw.json" # Display results echo "$BENCHMARK_JSON" | python3 -c " import json, sys d = json.load(sys.stdin) print(f' Files measured: {d[\"files_measured\"]}') print(f' Total raw tokens: {d[\"total_raw_tokens\"]:,}') print() print(' Per-language results:') print(f' {\"Lang\":<8s} {\"Files\":>5s} {\"Raw Tok\":>9s} {\"Best Mode\":<12s} {\"Compressed\":>10s} {\"Savings\":>8s}') print(f' {\"─\"*8} {\"─\"*5} {\"─\"*9} {\"─\"*12} {\"─\"*10} {\"─\"*8}') for l in d.get('languages', []): print(f' {l[\"ext\"]:<8s} {l[\"count\"]:>5d} {l[\"total_tokens\"]:>9,} {l[\"best_mode\"]:<12s} {l[\"best_mode_tokens\"]:>10,} {l[\"best_savings_pct\"]:>7.1f}%') print() print(' Mode performance:') for ms in d.get('mode_summaries', []): q = f'{ms[\"avg_preservation\"]:.0f}%' if ms.get('avg_preservation', 0) > 0 else 'N/A' print(f' {ms[\"mode\"]:15s} savings: {ms[\"avg_savings_pct\"]:>5.1f}% quality: {q:>5s}') " echo "" # ═══════════════════════════════════════════════════════ # SESSION SIMULATION # ═══════════════════════════════════════════════════════ echo "╔═══ Session Simulation (30-min coding) ═══╗" echo "" # Display human-readable benchmark (includes session sim) cd "$PROJECT_ROOT" && "$LEAN_CTX" benchmark run . 2>/dev/null | grep -A10 "Session Simulation" echo "" # ═══════════════════════════════════════════════════════ # CUMULATIVE PRODUCTION STATS # ═══════════════════════════════════════════════════════ echo "╔═══ Cumulative Production Stats ═══╗" echo "" cd "$PROJECT_ROOT" && "$LEAN_CTX" gain 2>/dev/null || echo " (no production data available)" echo "" # ═══════════════════════════════════════════════════════ # SAVE FINAL COMBINED RESULTS # ═══════════════════════════════════════════════════════ # Combine CLI + benchmark into final JSON echo "$BENCHMARK_JSON" | python3 -c " import json, sys d = json.load(sys.stdin) results = { 'version': '2.0', 'lean_ctx_version': '3.6.6', 'date': '$(date -u +%Y-%m-%dT%H:%M:%SZ)', 'tokenizer': 'tiktoken cl100k_base (exact)', 'project': 'lean-ctx (Rust/TS/Python, ~50K LOC)', 'cli_compression': { 'commands_tested': $CLI_COUNT, 'raw_tokens': $CLI_TOTAL_RAW, 'compressed_tokens': $CLI_TOTAL_COMPRESSED, 'savings_pct': round(($CLI_TOTAL_RAW - $CLI_TOTAL_COMPRESSED) * 100 / max($CLI_TOTAL_RAW, 1), 1) }, 'file_read_compression': { 'files_measured': d['files_measured'], 'total_raw_tokens': d['total_raw_tokens'], 'languages': d.get('languages', []), 'mode_summaries': d.get('mode_summaries', []) }, 'session_simulation': d.get('session_simulation', {}), 'methodology': { 'token_counting': 'tiktoken cl100k_base via Rust bindings (exact counts)', 'best_mode_selection': 'Only modes producing >0 tokens qualify (0 = data loss, excluded)', 'quality_metric': 'Semantic preservation via key-symbol retention', 'reproducibility': 'lean-ctx benchmark run /path --json' } } print(json.dumps(results, indent=2)) " > "$RESULTS_DIR/benchmark-results.json" echo "═══════════════════════════════════════════════════" echo " Results saved to: $RESULTS_DIR/benchmark-results.json" echo " Raw data: $RESULTS_DIR/benchmark-raw.json" echo "═══════════════════════════════════════════════════"