chore: import upstream snapshot with attribution
Python CI / Python bindings (ubuntu-latest) (push) Failing after 0s
Build & Publish / Build Neovim aarch64-linux-android (push) Failing after 0s
Build & Publish / Build Neovim aarch64-unknown-linux-gnu (push) Failing after 0s
Build & Publish / Build MCP x86_64-unknown-linux-musl (push) Failing after 1s
Build & Publish / Build Python wheels aarch64 (ubuntu-latest) (push) Failing after 0s
Build & Publish / Build Python wheels x86_64 (ubuntu-latest) (push) Failing after 1s
Build & Publish / Build Python sdist (push) Failing after 1s
Rust CI / Fuzz Tests (ubuntu-latest) (push) Failing after 1s
Rust CI / Build i686-unknown-linux-gnu (push) Failing after 0s
Rust CI / cargo clippy (push) Failing after 1s
e2e Tests / e2e (ubuntu-latest) (push) Failing after 1s
Lua CI / luacheck lint (push) Failing after 1s
Build & Publish / Build Neovim aarch64-unknown-linux-musl (push) Failing after 1s
Build & Publish / Build Neovim x86_64-unknown-linux-gnu (push) Failing after 0s
Build & Publish / Build Neovim x86_64-unknown-linux-musl (push) Failing after 1s
Build & Publish / Build C FFI aarch64-linux-android (push) Failing after 0s
Build & Publish / Build C FFI aarch64-unknown-linux-gnu (push) Failing after 0s
Build & Publish / Build C FFI aarch64-unknown-linux-musl (push) Failing after 1s
Build & Publish / Build C FFI x86_64-unknown-linux-gnu (push) Failing after 0s
Build & Publish / Build C FFI x86_64-unknown-linux-musl (push) Failing after 1s
Build & Publish / Build MCP aarch64-unknown-linux-gnu (push) Failing after 1s
Build & Publish / Build MCP aarch64-unknown-linux-musl (push) Failing after 0s
Build & Publish / Build MCP x86_64-unknown-linux-gnu (push) Failing after 1s
Spelling / Spell Check with Typos (push) Failing after 0s
Rust CI / Test (ubuntu-latest) (push) Failing after 0s
Rust CI / cargo fmt (push) Failing after 0s
Stylua / Check lua files using Stylua (push) Failing after 1s
Lua CI / lua-language-server type check (push) Failing after 12m29s
e2e Tests / e2e (alpine-musl) (push) Successful in 57m31s
e2e Tests / e2e (macos-latest) (push) Has been cancelled
Build & Publish / Build C FFI aarch64-apple-darwin (push) Has been cancelled
Rust CI / Test (windows-latest) (push) Has been cancelled
e2e Tests / e2e (windows-latest) (push) Has been cancelled
Nix CI / check (push) Has been cancelled
Python CI / Python bindings (macos-latest) (push) Has been cancelled
Python CI / Python bindings (windows-latest) (push) Has been cancelled
Build & Publish / Build Neovim aarch64-apple-darwin (push) Has been cancelled
Build & Publish / Build Neovim aarch64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build Neovim x86_64-apple-darwin (push) Has been cancelled
Build & Publish / Build Neovim x86_64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build C FFI aarch64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build C FFI x86_64-apple-darwin (push) Has been cancelled
Build & Publish / Build C FFI x86_64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build MCP aarch64-apple-darwin (push) Has been cancelled
Build & Publish / Build MCP aarch64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build MCP x86_64-apple-darwin (push) Has been cancelled
Build & Publish / Build MCP x86_64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build Python wheels aarch64 (macos-latest) (push) Has been cancelled
Build & Publish / Build Python wheels x86_64 (macos-latest) (push) Has been cancelled
Build & Publish / Build Python wheels x86_64 (windows-latest) (push) Has been cancelled
Build & Publish / Release (push) Has been cancelled
Build & Publish / Publish Python wheels to PyPI (push) Has been cancelled
Build & Publish / Publish Rust crates (push) Has been cancelled
Build & Publish / Publish npm packages (push) Has been cancelled
Rust CI / Fuzz Tests (windows-latest) (push) Has been cancelled
Rust CI / Test (macos-latest) (push) Has been cancelled
Rust CI / Fuzz Tests (macos-latest) (push) Has been cancelled
Python CI / Python bindings (ubuntu-latest) (push) Failing after 0s
Build & Publish / Build Neovim aarch64-linux-android (push) Failing after 0s
Build & Publish / Build Neovim aarch64-unknown-linux-gnu (push) Failing after 0s
Build & Publish / Build MCP x86_64-unknown-linux-musl (push) Failing after 1s
Build & Publish / Build Python wheels aarch64 (ubuntu-latest) (push) Failing after 0s
Build & Publish / Build Python wheels x86_64 (ubuntu-latest) (push) Failing after 1s
Build & Publish / Build Python sdist (push) Failing after 1s
Rust CI / Fuzz Tests (ubuntu-latest) (push) Failing after 1s
Rust CI / Build i686-unknown-linux-gnu (push) Failing after 0s
Rust CI / cargo clippy (push) Failing after 1s
e2e Tests / e2e (ubuntu-latest) (push) Failing after 1s
Lua CI / luacheck lint (push) Failing after 1s
Build & Publish / Build Neovim aarch64-unknown-linux-musl (push) Failing after 1s
Build & Publish / Build Neovim x86_64-unknown-linux-gnu (push) Failing after 0s
Build & Publish / Build Neovim x86_64-unknown-linux-musl (push) Failing after 1s
Build & Publish / Build C FFI aarch64-linux-android (push) Failing after 0s
Build & Publish / Build C FFI aarch64-unknown-linux-gnu (push) Failing after 0s
Build & Publish / Build C FFI aarch64-unknown-linux-musl (push) Failing after 1s
Build & Publish / Build C FFI x86_64-unknown-linux-gnu (push) Failing after 0s
Build & Publish / Build C FFI x86_64-unknown-linux-musl (push) Failing after 1s
Build & Publish / Build MCP aarch64-unknown-linux-gnu (push) Failing after 1s
Build & Publish / Build MCP aarch64-unknown-linux-musl (push) Failing after 0s
Build & Publish / Build MCP x86_64-unknown-linux-gnu (push) Failing after 1s
Spelling / Spell Check with Typos (push) Failing after 0s
Rust CI / Test (ubuntu-latest) (push) Failing after 0s
Rust CI / cargo fmt (push) Failing after 0s
Stylua / Check lua files using Stylua (push) Failing after 1s
Lua CI / lua-language-server type check (push) Failing after 12m29s
e2e Tests / e2e (alpine-musl) (push) Successful in 57m31s
e2e Tests / e2e (macos-latest) (push) Has been cancelled
Build & Publish / Build C FFI aarch64-apple-darwin (push) Has been cancelled
Rust CI / Test (windows-latest) (push) Has been cancelled
e2e Tests / e2e (windows-latest) (push) Has been cancelled
Nix CI / check (push) Has been cancelled
Python CI / Python bindings (macos-latest) (push) Has been cancelled
Python CI / Python bindings (windows-latest) (push) Has been cancelled
Build & Publish / Build Neovim aarch64-apple-darwin (push) Has been cancelled
Build & Publish / Build Neovim aarch64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build Neovim x86_64-apple-darwin (push) Has been cancelled
Build & Publish / Build Neovim x86_64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build C FFI aarch64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build C FFI x86_64-apple-darwin (push) Has been cancelled
Build & Publish / Build C FFI x86_64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build MCP aarch64-apple-darwin (push) Has been cancelled
Build & Publish / Build MCP aarch64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build MCP x86_64-apple-darwin (push) Has been cancelled
Build & Publish / Build MCP x86_64-pc-windows-msvc (push) Has been cancelled
Build & Publish / Build Python wheels aarch64 (macos-latest) (push) Has been cancelled
Build & Publish / Build Python wheels x86_64 (macos-latest) (push) Has been cancelled
Build & Publish / Build Python wheels x86_64 (windows-latest) (push) Has been cancelled
Build & Publish / Release (push) Has been cancelled
Build & Publish / Publish Python wheels to PyPI (push) Has been cancelled
Build & Publish / Publish Rust crates (push) Has been cancelled
Build & Publish / Publish npm packages (push) Has been cancelled
Rust CI / Fuzz Tests (windows-latest) (push) Has been cancelled
Rust CI / Test (macos-latest) (push) Has been cancelled
Rust CI / Fuzz Tests (macos-latest) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Analyze benchmark results across all concepts and iterations."""
|
||||
import json, os, glob, sys
|
||||
|
||||
results_dir = sys.argv[1] if len(sys.argv) > 1 else "scripts/benchmark-results"
|
||||
|
||||
# Concept names (order matches benchmark-claude.sh)
|
||||
CONCEPT_NAMES = [
|
||||
"fuzzy-function-search",
|
||||
"api-endpoint-discovery",
|
||||
"cross-service-config",
|
||||
"test-file-discovery",
|
||||
"error-type-definition",
|
||||
"database-model-search",
|
||||
"auth-flow-tracing",
|
||||
"todo-tech-debt",
|
||||
"cross-language-pattern",
|
||||
"broad-pattern-search",
|
||||
]
|
||||
|
||||
def load_iter_results(concept_name, mode):
|
||||
results = []
|
||||
for i in range(1, 100):
|
||||
path = os.path.join(results_dir, f"{concept_name}-{mode}-iter{i}.json")
|
||||
if not os.path.exists(path):
|
||||
break
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
if data.get("total_cost_usd", 0) > 0:
|
||||
results.append(data)
|
||||
except:
|
||||
pass
|
||||
# Also check the non-iter file as fallback
|
||||
if not results:
|
||||
path = os.path.join(results_dir, f"{concept_name}-{mode}.json")
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
if data.get("total_cost_usd", 0) > 0:
|
||||
results.append(data)
|
||||
except:
|
||||
pass
|
||||
return results
|
||||
|
||||
def load_stream_trace(concept_name, mode, iteration):
|
||||
"""Load tool call trace from stream file."""
|
||||
path = os.path.join(results_dir, f"{concept_name}-{mode}-iter{iteration}.stream.jsonl")
|
||||
if not os.path.exists(path):
|
||||
path = os.path.join(results_dir, f"{concept_name}-{mode}.stream.jsonl")
|
||||
if not os.path.exists(path):
|
||||
return []
|
||||
|
||||
tool_calls = []
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(line)
|
||||
except:
|
||||
continue
|
||||
if msg.get("type") == "assistant" and "message" in msg:
|
||||
for block in msg["message"].get("content", []):
|
||||
if isinstance(block, dict) and block.get("type") == "tool_use":
|
||||
name = block.get("name", "?")
|
||||
inp = block.get("input", {})
|
||||
summary = ""
|
||||
if "query" in inp:
|
||||
summary = inp["query"]
|
||||
elif "pattern" in inp:
|
||||
summary = inp["pattern"]
|
||||
elif "patterns" in inp:
|
||||
summary = str(inp["patterns"])
|
||||
elif "file_path" in inp:
|
||||
summary = inp["file_path"].split("/")[-1]
|
||||
elif "command" in inp:
|
||||
summary = inp["command"][:50]
|
||||
tool_calls.append({"name": name, "summary": summary})
|
||||
return tool_calls
|
||||
|
||||
|
||||
print("=" * 90)
|
||||
print(" FFF MCP vs Native — All Concepts Summary")
|
||||
print("=" * 90)
|
||||
|
||||
total_fff_cost = 0
|
||||
total_nat_cost = 0
|
||||
fff_wins = 0
|
||||
nat_wins = 0
|
||||
ties = 0
|
||||
|
||||
concept_data = []
|
||||
|
||||
for i, name in enumerate(CONCEPT_NAMES):
|
||||
fff_results = load_iter_results(name, "fff")
|
||||
nat_results = load_iter_results(name, "native")
|
||||
|
||||
if not fff_results and not nat_results:
|
||||
continue
|
||||
|
||||
fff_avg_cost = sum(r.get("total_cost_usd", 0) for r in fff_results) / max(len(fff_results), 1)
|
||||
nat_avg_cost = sum(r.get("total_cost_usd", 0) for r in nat_results) / max(len(nat_results), 1)
|
||||
fff_avg_turns = sum(r.get("num_turns", 0) for r in fff_results) / max(len(fff_results), 1)
|
||||
nat_avg_turns = sum(r.get("num_turns", 0) for r in nat_results) / max(len(nat_results), 1)
|
||||
fff_avg_wall = sum(r.get("wall_ms", 0) for r in fff_results) / max(len(fff_results), 1) / 1000
|
||||
nat_avg_wall = sum(r.get("wall_ms", 0) for r in nat_results) / max(len(nat_results), 1) / 1000
|
||||
|
||||
if fff_avg_cost < nat_avg_cost * 0.95:
|
||||
winner = "FFF"
|
||||
fff_wins += 1
|
||||
elif nat_avg_cost < fff_avg_cost * 0.95:
|
||||
winner = "NATIVE"
|
||||
nat_wins += 1
|
||||
else:
|
||||
winner = "TIE"
|
||||
ties += 1
|
||||
|
||||
total_fff_cost += fff_avg_cost
|
||||
total_nat_cost += nat_avg_cost
|
||||
|
||||
concept_data.append({
|
||||
"num": i + 1,
|
||||
"name": name,
|
||||
"fff_cost": fff_avg_cost,
|
||||
"nat_cost": nat_avg_cost,
|
||||
"fff_turns": fff_avg_turns,
|
||||
"nat_turns": nat_avg_turns,
|
||||
"fff_wall": fff_avg_wall,
|
||||
"nat_wall": nat_avg_wall,
|
||||
"fff_n": len(fff_results),
|
||||
"nat_n": len(nat_results),
|
||||
"winner": winner,
|
||||
})
|
||||
|
||||
# Print table
|
||||
print(f"\n {'#':>2} {'Concept':<28} {'FFF $':>8} {'Nat $':>8} {'FFF T':>5} {'Nat T':>5} {'FFF s':>6} {'Nat s':>6} {'N':>3} {'Winner':>8}")
|
||||
print(f" {'─'*2} {'─'*28} {'─'*8} {'─'*8} {'─'*5} {'─'*5} {'─'*6} {'─'*6} {'─'*3} {'─'*8}")
|
||||
|
||||
for d in concept_data:
|
||||
savings = (1 - d["fff_cost"] / d["nat_cost"]) * 100 if d["nat_cost"] > 0 else 0
|
||||
print(f" {d['num']:>2} {d['name']:<28} ${d['fff_cost']:.4f} ${d['nat_cost']:.4f} {d['fff_turns']:>5.1f} {d['nat_turns']:>5.1f} {d['fff_wall']:>5.0f}s {d['nat_wall']:>5.0f}s {d['fff_n']:>3} {d['winner']:>8}")
|
||||
|
||||
print(f"\n Score: FFF {fff_wins} | Native {nat_wins} | Tie {ties}")
|
||||
print(f" Total avg cost: FFF ${total_fff_cost:.4f} | Native ${total_nat_cost:.4f}")
|
||||
if total_nat_cost > 0:
|
||||
print(f" Overall savings: {(1 - total_fff_cost / total_nat_cost) * 100:+.1f}%")
|
||||
|
||||
# Show problematic concepts (where native wins by >20%)
|
||||
print(f"\n{'─' * 90}")
|
||||
print(" Concepts where fff loses (native wins by >5%):")
|
||||
for d in concept_data:
|
||||
if d["winner"] == "NATIVE":
|
||||
pct = (d["fff_cost"] / d["nat_cost"] - 1) * 100
|
||||
print(f" #{d['num']} {d['name']}: fff is {pct:+.0f}% more expensive")
|
||||
# Show tool traces for the worst iteration
|
||||
traces = load_stream_trace(d["name"], "fff", 1)
|
||||
if traces:
|
||||
print(f" fff trace: {' → '.join(t['name'].replace('mcp__fff__','') for t in traces)}")
|
||||
traces = load_stream_trace(d["name"], "native", 1)
|
||||
if traces:
|
||||
print(f" nat trace: {' → '.join(t['name'] for t in traces)}")
|
||||
|
||||
print()
|
||||
@@ -0,0 +1,816 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Benchmark: fff MCP vs Claude Code native tools on real search tasks
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/benchmark-claude.sh [concept_number] [--fff-only | --native-only]
|
||||
#
|
||||
# Runs real Claude Code instances against ~/dev/lightsource:
|
||||
# - With fff MCP tools (frecency-ranked, fuzzy search)
|
||||
# - With native tools only (Glob, Grep, Read)
|
||||
# Then compares: tokens, cost, turns, and whether the right file was found.
|
||||
#
|
||||
# Requirements:
|
||||
# - claude CLI in PATH
|
||||
# - ~/dev/lightsource exists
|
||||
# - fff MCP server built (cargo build --release, binary at target/release/fff-mcp)
|
||||
#
|
||||
# Auth: The script inherits YOUR shell environment. If you use AWS Bedrock,
|
||||
# make sure your AWS credentials are exported before running.
|
||||
# Run `claude --print -p "hello"` first to verify auth works.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
||||
LIGHTSOURCE="$HOME/dev/lightsource"
|
||||
RESULTS_DIR="$SCRIPT_DIR/benchmark-results"
|
||||
MAX_TURNS=10
|
||||
TIMEOUT_SEC=300 # 5 min per concept per mode
|
||||
MODEL="us.anthropic.claude-opus-4-6-v1"
|
||||
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
# Write MCP config to temp file to avoid shell quoting issues.
|
||||
# Both modes (fff and native) connect the fff MCP so context overhead is identical.
|
||||
FFF_MCP_FILE=$(mktemp)
|
||||
trap "rm -f $FFF_MCP_FILE" EXIT
|
||||
|
||||
cat > "$FFF_MCP_FILE" <<EOF
|
||||
{"mcpServers":{"fff":{"type":"stdio","command":"$PROJECT_ROOT/target/release/fff-mcp","args":[]}}}
|
||||
EOF
|
||||
|
||||
# ─── PREFLIGHT CHECK ──────────────────────────────────────────────────────────
|
||||
|
||||
echo "Preflight check..."
|
||||
if ! command -v claude &>/dev/null; then
|
||||
echo "ERROR: claude CLI not found in PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -d "$LIGHTSOURCE" ]]; then
|
||||
echo "ERROR: $LIGHTSOURCE does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Quick auth test — must clear nesting env vars, cd to lightsource, use </dev/null
|
||||
AUTH_TEST=$(cd "$LIGHTSOURCE" && env -u CLAUDECODE -u CLAUDE_CODE_ENTRYPOINT \
|
||||
timeout 60s claude --print --output-format json -p "say ok" --max-turns 1 \
|
||||
--mcp-config "$FFF_MCP_FILE" --strict-mcp-config </dev/null 2>&1 || true)
|
||||
if [[ -z "$AUTH_TEST" ]] || echo "$AUTH_TEST" | grep -q '"is_error":true' 2>/dev/null; then
|
||||
echo "ERROR: Claude auth failed. Test output:"
|
||||
echo "$AUTH_TEST" | head -5
|
||||
echo ""
|
||||
echo "If using AWS Bedrock, make sure your AWS credentials are exported:"
|
||||
echo " export AWS_ACCESS_KEY_ID=..."
|
||||
echo " export AWS_SECRET_ACCESS_KEY=..."
|
||||
echo " export AWS_SESSION_TOKEN=..."
|
||||
echo ""
|
||||
echo "Or run: aws sso login"
|
||||
exit 1
|
||||
fi
|
||||
echo " Auth OK"
|
||||
echo ""
|
||||
|
||||
# ─── 10 SEARCH CONCEPTS ───────────────────────────────────────────────────────
|
||||
|
||||
declare -a PROMPTS
|
||||
declare -a TARGETS
|
||||
declare -a NAMES
|
||||
|
||||
NAMES[1]="fuzzy-function-search"
|
||||
PROMPTS[1]="Find the function that loads metadata for an InProgressQuote in the lightsource codebase. Show me the function signature and which file it's in."
|
||||
TARGETS[1]="quotes/storage/db/src/model/quote.rs"
|
||||
|
||||
NAMES[2]="api-endpoint-discovery"
|
||||
PROMPTS[2]="Find the GraphQL mutation that handles user file uploads (the prepare upload step). Show me the function and its file path."
|
||||
TARGETS[2]="user_files_service/graphql/src/mutation.rs"
|
||||
|
||||
NAMES[3]="cross-service-config"
|
||||
PROMPTS[3]="Find where QuotesServiceClient is defined as a struct and how it's constructed. Show me the struct definition and its file."
|
||||
TARGETS[3]="quotes_service_client"
|
||||
|
||||
NAMES[4]="test-file-discovery"
|
||||
PROMPTS[4]="Find the test file for virtual expression manifests in the quotes engine. Show me the file path and list what tests are in it."
|
||||
TARGETS[4]="virtual_expression_manifest_test"
|
||||
|
||||
NAMES[5]="error-type-definition"
|
||||
PROMPTS[5]="Find where the custom Error type with variants like not_found and permission_denied is defined in the common/error crate. Show me the enum or struct definition."
|
||||
TARGETS[5]="common/error"
|
||||
|
||||
NAMES[6]="database-model-search"
|
||||
PROMPTS[6]="Find the Diesel ORM model struct for InProgressQuote — the actual struct definition with its derives, not usages. Show me the struct and its file path."
|
||||
TARGETS[6]="quotes/storage/db/src/model/quote.rs"
|
||||
|
||||
NAMES[7]="auth-flow-tracing"
|
||||
PROMPTS[7]="Find where ActorAuth is defined and trace how it's used in service GraphQL contexts. Show me the definition and one example of it being extracted in a resolver."
|
||||
TARGETS[7]="actor_auth"
|
||||
|
||||
NAMES[8]="todo-tech-debt"
|
||||
PROMPTS[8]="Find TODO comments tagged with github issues numbers (like #... or similar patterns) in the quotes-related code. Show me a few examples with their file paths."
|
||||
TARGETS[8]="TODO"
|
||||
|
||||
NAMES[9]="cross-language-pattern"
|
||||
PROMPTS[9]="Find code related to QuoteBuilder across both Rust backend and TypeScript frontend. Show me one example from each language."
|
||||
TARGETS[9]="QuoteBuilder"
|
||||
|
||||
NAMES[10]="broad-pattern-search"
|
||||
PROMPTS[10]="Find the main GraphQL query resolvers for sourcing projects — specifically the resolver that loads a single sourcing project by ID. Show me the resolver function and file."
|
||||
TARGETS[10]="sourcing_project"
|
||||
|
||||
NAMES[11]="file-by-name-lookup"
|
||||
PROMPTS[11]="What files exist in this repository related to 'quote_builder'? List 10 paths from frontend and 10 from backend."
|
||||
TARGETS[11]="quote_builder"
|
||||
|
||||
# NOTE: Concept 11 tests file-by-name lookup. The model strongly prefers native Glob
|
||||
# over find_files due to Claude Code's system prompt. find_files would be faster here
|
||||
# (fuzzy: 'tsconfig sourcing' → 1 call) but the model won't use it unprompted.
|
||||
|
||||
# ─── HELPER FUNCTIONS ──────────────────────────────────────────────────────────
|
||||
|
||||
millis() {
|
||||
python3 -c 'import time; print(int(time.time()*1000))'
|
||||
}
|
||||
|
||||
run_claude() {
|
||||
local mode="$1" # "fff" or "native"
|
||||
local concept="$2"
|
||||
local raw_prompt="${PROMPTS[$concept]}"
|
||||
local outfile="$RESULTS_DIR/${NAMES[$concept]}-${mode}.json"
|
||||
|
||||
# Both modes connect fff MCP so context overhead is identical.
|
||||
# The prompt prefix steers which tools Claude actually uses.
|
||||
local mcp_args=(--mcp-config "$FFF_MCP_FILE" --strict-mcp-config)
|
||||
# (tool_args removed — both modes use identical MCP config, prompt steers tool choice)
|
||||
|
||||
local reasoning_instruction="IMPORTANT: Before EVERY tool call, write 1-2 sentences explaining your reasoning: why you chose this specific tool, what query/pattern you picked and why, what you expect to find, and if this is a follow-up, what the previous result told you that led to this next step."
|
||||
|
||||
local prompt
|
||||
if [[ "$mode" == "fff" ]]; then
|
||||
prompt="Use fff tools (grep, find_files, multi_grep) instead of native Glob/Grep.
|
||||
|
||||
$reasoning_instruction
|
||||
|
||||
$raw_prompt"
|
||||
else
|
||||
prompt="IMPORTANT: For file search and content search, use ONLY the native tools (Glob, Grep, Read). Do NOT use any mcp__fff__* tools. Ignore the fff MCP server entirely.
|
||||
|
||||
$reasoning_instruction
|
||||
|
||||
$raw_prompt"
|
||||
fi
|
||||
|
||||
local model_args=()
|
||||
if [[ -n "$MODEL" ]]; then
|
||||
model_args=(--model "$MODEL")
|
||||
fi
|
||||
|
||||
local errfile="$RESULTS_DIR/${NAMES[$concept]}-${mode}.stderr"
|
||||
local streamfile="$RESULTS_DIR/${NAMES[$concept]}-${mode}.stream.jsonl"
|
||||
|
||||
echo " Running [$mode] concept $concept: ${NAMES[$concept]} (timeout ${TIMEOUT_SEC}s)..."
|
||||
|
||||
local start_time
|
||||
start_time=$(millis)
|
||||
|
||||
# Capture stream-json for per-turn analysis, then extract the final result.
|
||||
# IMPORTANT: </dev/null prevents stdin blocking, cd to LIGHTSOURCE so Claude's
|
||||
# tools work in the right directory.
|
||||
(
|
||||
cd "$LIGHTSOURCE"
|
||||
timeout "${TIMEOUT_SEC}s" env -u CLAUDECODE -u CLAUDE_CODE_ENTRYPOINT \
|
||||
claude \
|
||||
--print \
|
||||
--verbose \
|
||||
--output-format stream-json \
|
||||
--max-turns "$MAX_TURNS" \
|
||||
--max-budget-usd 0.50 \
|
||||
--dangerously-skip-permissions \
|
||||
"${model_args[@]}" \
|
||||
"${mcp_args[@]}" \
|
||||
-p "$prompt" \
|
||||
</dev/null \
|
||||
> "$streamfile" 2>"$errfile"
|
||||
) || {
|
||||
local exit_code=$?
|
||||
if [[ $exit_code -eq 124 ]]; then
|
||||
echo " TIMEOUT after ${TIMEOUT_SEC}s"
|
||||
echo "{\"type\":\"result\",\"is_error\":true,\"result\":\"TIMEOUT after ${TIMEOUT_SEC}s\",\"num_turns\":0,\"total_cost_usd\":0,\"duration_ms\":0,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}" > "$outfile"
|
||||
return
|
||||
elif [[ ! -s "$streamfile" ]]; then
|
||||
echo " FAILED (exit $exit_code)"
|
||||
local stderr_msg
|
||||
stderr_msg=$(head -3 "$errfile" 2>/dev/null | tr '\n' ' ')
|
||||
echo "{\"type\":\"result\",\"is_error\":true,\"result\":\"Process failed (exit $exit_code): $stderr_msg\",\"num_turns\":0,\"total_cost_usd\":0,\"duration_ms\":0,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}" > "$outfile"
|
||||
fi
|
||||
}
|
||||
|
||||
# Print stderr if non-empty (helps debugging)
|
||||
if [[ -s "$errfile" ]]; then
|
||||
echo " stderr: $(head -1 "$errfile")"
|
||||
fi
|
||||
|
||||
# Extract final result JSON from stream (last line with type=result)
|
||||
if [[ -s "$streamfile" ]]; then
|
||||
grep '"type":"result"' "$streamfile" | tail -1 > "$outfile" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
local end_time
|
||||
end_time=$(millis)
|
||||
local wall_ms=$(( end_time - start_time ))
|
||||
|
||||
# Inject wall time into the JSON
|
||||
if [[ -f "$outfile" ]] && [[ -s "$outfile" ]]; then
|
||||
local tmp
|
||||
tmp=$(mktemp)
|
||||
jq --argjson wall "$wall_ms" '. + {wall_ms: $wall}' "$outfile" > "$tmp" 2>/dev/null && mv "$tmp" "$outfile" || rm -f "$tmp"
|
||||
fi
|
||||
|
||||
# Quick status line
|
||||
local cost turns found_str cost_fmt
|
||||
cost=$(jq -r '.total_cost_usd // 0' "$outfile" 2>/dev/null || echo "0")
|
||||
cost_fmt=$(printf '%.4f' "$cost" 2>/dev/null || echo "$cost")
|
||||
turns=$(jq -r '.num_turns // 0' "$outfile" 2>/dev/null || echo "?")
|
||||
local err=$(jq -r '.is_error // false' "$outfile" 2>/dev/null || echo "?")
|
||||
if [[ "$err" == "true" ]]; then
|
||||
found_str="ERROR"
|
||||
else
|
||||
found_str="ok"
|
||||
fi
|
||||
echo " Done in $((wall_ms/1000))s | \$$cost_fmt | ${turns} turns | $found_str"
|
||||
|
||||
# ── Per-turn tool call analysis ──
|
||||
if [[ -s "$streamfile" ]]; then
|
||||
echo ""
|
||||
echo " ┌─ Tool call trace [$mode] ─────────────────────────────────────────"
|
||||
# Extract tool_use events and tool_result sizes from stream
|
||||
python3 - "$streamfile" <<'PYEOF'
|
||||
import json, sys, textwrap
|
||||
|
||||
stream_file = sys.argv[1]
|
||||
|
||||
# Ordered list of events: ("text", text) | ("tool", name, summary, id) | ("result", id, size)
|
||||
events = []
|
||||
result_sizes = {} # tool_use_id -> content_length
|
||||
|
||||
with open(stream_file) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(line)
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
|
||||
msg_type = msg.get("type", "")
|
||||
content_blocks = []
|
||||
|
||||
if msg_type == "assistant" and "message" in msg:
|
||||
content_blocks = msg["message"].get("content", [])
|
||||
elif msg_type == "user" and "message" in msg:
|
||||
content_blocks = msg["message"].get("content", [])
|
||||
|
||||
for block in content_blocks:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
|
||||
# Assistant reasoning text
|
||||
if block.get("type") == "text" and msg_type == "assistant":
|
||||
text = block.get("text", "").strip()
|
||||
if text:
|
||||
events.append(("text", text))
|
||||
|
||||
# Tool use (Claude calling a tool)
|
||||
if block.get("type") == "tool_use":
|
||||
name = block.get("name", "?")
|
||||
inp = block.get("input", {})
|
||||
if "query" in inp:
|
||||
summary = f'query="{inp["query"]}"'
|
||||
elif "pattern" in inp:
|
||||
summary = f'pattern="{inp["pattern"]}"'
|
||||
elif "patterns" in inp:
|
||||
summary = f'patterns={json.dumps(inp["patterns"])}'
|
||||
elif "file_path" in inp:
|
||||
summary = f'file="{inp["file_path"][-60:]}"'
|
||||
elif "path" in inp:
|
||||
summary = f'path="{inp["path"][-60:]}"'
|
||||
elif "command" in inp:
|
||||
summary = f'cmd="{inp["command"][:70]}"'
|
||||
else:
|
||||
summary = str(inp)[:70]
|
||||
events.append(("tool", name, summary, block.get("id", "")))
|
||||
|
||||
# Tool result (response back from tool)
|
||||
if block.get("type") == "tool_result":
|
||||
tid = block.get("tool_use_id", "")
|
||||
content = block.get("content", "")
|
||||
if isinstance(content, list):
|
||||
total_len = sum(len(c.get("text", "")) for c in content if isinstance(c, dict))
|
||||
elif isinstance(content, str):
|
||||
total_len = len(content)
|
||||
else:
|
||||
total_len = len(str(content))
|
||||
result_sizes[tid] = total_len
|
||||
|
||||
# Print trace with reasoning
|
||||
tool_num = 0
|
||||
last_was_text = False
|
||||
for event in events:
|
||||
if event[0] == "text":
|
||||
text = event[1]
|
||||
# Truncate long reasoning, skip final answer blocks (contain code fences)
|
||||
if "```" in text:
|
||||
# Final answer with code — just show first line
|
||||
first_line = text.split("\n")[0].strip()
|
||||
if first_line:
|
||||
text = first_line[:120] + ("..." if len(first_line) > 120 else "")
|
||||
else:
|
||||
continue
|
||||
elif len(text) > 300:
|
||||
text = text[:297] + "..."
|
||||
wrapped = textwrap.wrap(text, width=90)
|
||||
if not last_was_text:
|
||||
print(" |")
|
||||
for wline in wrapped:
|
||||
print(f" | 💭 {wline}")
|
||||
last_was_text = True
|
||||
elif event[0] == "tool":
|
||||
_, name, summary, tid = event
|
||||
tool_num += 1
|
||||
rsize = result_sizes.get(tid, -1)
|
||||
size_str = f" -> {rsize:,} chars" if rsize >= 0 else ""
|
||||
print(f" | {tool_num:2d}. {name:25s} {summary[:50]:50s}{size_str}")
|
||||
last_was_text = False
|
||||
|
||||
if tool_num == 0:
|
||||
print(" | (no tool calls captured)")
|
||||
|
||||
print(f" |")
|
||||
print(f" | Total: {tool_num} tool calls")
|
||||
PYEOF
|
||||
echo " └──────────────────────────────────────────────────────────────────"
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
parse_result() {
|
||||
local jsonfile="$1"
|
||||
local target="$2"
|
||||
|
||||
if [[ ! -f "$jsonfile" ]] || [[ ! -s "$jsonfile" ]]; then
|
||||
echo "0|0|0|0|false|false"
|
||||
return
|
||||
fi
|
||||
|
||||
local cost_usd num_turns duration_ms wall_ms is_error result
|
||||
cost_usd=$(jq -r '.total_cost_usd // 0' "$jsonfile" 2>/dev/null || echo "0")
|
||||
num_turns=$(jq -r '.num_turns // 0' "$jsonfile" 2>/dev/null || echo "0")
|
||||
duration_ms=$(jq -r '.duration_ms // 0' "$jsonfile" 2>/dev/null || echo "0")
|
||||
wall_ms=$(jq -r '.wall_ms // 0' "$jsonfile" 2>/dev/null || echo "0")
|
||||
is_error=$(jq -r '.is_error // false' "$jsonfile" 2>/dev/null || echo "false")
|
||||
result=$(jq -r '.result // ""' "$jsonfile" 2>/dev/null || echo "")
|
||||
|
||||
local found="false"
|
||||
if echo "$result" | grep -qi "$target" 2>/dev/null; then
|
||||
found="true"
|
||||
fi
|
||||
|
||||
echo "${cost_usd}|${num_turns}|${duration_ms}|${wall_ms}|${is_error}|${found}"
|
||||
}
|
||||
|
||||
print_comparison() {
|
||||
local concept="$1"
|
||||
local name="${NAMES[$concept]}"
|
||||
local target="${TARGETS[$concept]}"
|
||||
|
||||
local fff_file="$RESULTS_DIR/${name}-fff.json"
|
||||
local native_file="$RESULTS_DIR/${name}-native.json"
|
||||
|
||||
local fff_data native_data
|
||||
fff_data=$(parse_result "$fff_file" "$target")
|
||||
native_data=$(parse_result "$native_file" "$target")
|
||||
|
||||
IFS='|' read -r fff_cost fff_turns fff_dur fff_wall fff_err fff_found <<< "$fff_data"
|
||||
IFS='|' read -r nat_cost nat_turns nat_dur nat_wall nat_err nat_found <<< "$native_data"
|
||||
|
||||
# Token counts from usage
|
||||
local fff_input fff_output nat_input nat_output
|
||||
fff_input=$(jq -r '.usage.input_tokens // 0' "$fff_file" 2>/dev/null || echo "0")
|
||||
fff_output=$(jq -r '.usage.output_tokens // 0' "$fff_file" 2>/dev/null || echo "0")
|
||||
nat_input=$(jq -r '.usage.input_tokens // 0' "$native_file" 2>/dev/null || echo "0")
|
||||
nat_output=$(jq -r '.usage.output_tokens // 0' "$native_file" 2>/dev/null || echo "0")
|
||||
local fff_tokens=$((fff_input + fff_output))
|
||||
local nat_tokens=$((nat_input + nat_output))
|
||||
|
||||
# Determine winner
|
||||
local winner="tie"
|
||||
if [[ "$fff_found" == "true" && "$nat_found" == "false" ]]; then
|
||||
winner="FFF"
|
||||
elif [[ "$fff_found" == "false" && "$nat_found" == "true" ]]; then
|
||||
winner="NATIVE"
|
||||
elif [[ "$fff_found" == "true" && "$nat_found" == "true" ]]; then
|
||||
# Both found — compare cost with 15% tolerance band for ties
|
||||
local ratio
|
||||
ratio=$(echo "scale=4; $fff_cost / $nat_cost" | bc 2>/dev/null || echo "1")
|
||||
# ratio < 0.85 means FFF is >15% cheaper → FFF wins
|
||||
# ratio > 1.15 means FFF is >15% more expensive → NATIVE wins
|
||||
# otherwise → tie
|
||||
local ratio_x100
|
||||
ratio_x100=$(echo "$ratio * 100" | bc 2>/dev/null | cut -d. -f1 || echo "100")
|
||||
if [[ "${ratio_x100:-100}" -lt 85 ]]; then
|
||||
winner="FFF"
|
||||
elif [[ "${ratio_x100:-100}" -gt 115 ]]; then
|
||||
winner="NATIVE"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Format costs to 4 decimal places
|
||||
local fff_cost_fmt nat_cost_fmt
|
||||
fff_cost_fmt=$(printf '%.4f' "$fff_cost" 2>/dev/null || echo "$fff_cost")
|
||||
nat_cost_fmt=$(printf '%.4f' "$nat_cost" 2>/dev/null || echo "$nat_cost")
|
||||
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " CONCEPT $concept: $name"
|
||||
echo " Target: $target"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
printf " %-12s │ %10s │ %6s │ %8s │ %8s │ %7s │ %7s\n" "" "Cost" "Turns" "Tokens" "Wall (s)" "Found?" "Error?"
|
||||
echo " ─────────────┼────────────┼────────┼──────────┼──────────┼─────────┼────────"
|
||||
printf " %-12s │ %10s │ %6s │ %8s │ %8s │ %7s │ %7s\n" \
|
||||
"fff MCP" "\$$fff_cost_fmt" "$fff_turns" "$fff_tokens" "$((fff_wall/1000))" "$fff_found" "$fff_err"
|
||||
printf " %-12s │ %10s │ %6s │ %8s │ %8s │ %7s │ %7s\n" \
|
||||
"Native" "\$$nat_cost_fmt" "$nat_turns" "$nat_tokens" "$((nat_wall/1000))" "$nat_found" "$nat_err"
|
||||
echo " ─────────────┴────────────┴────────┴──────────┴──────────┴─────────┴────────"
|
||||
|
||||
# Cost savings percentage
|
||||
if [[ "$nat_cost" != "0" ]]; then
|
||||
local cost_savings
|
||||
cost_savings=$(echo "scale=1; (1 - $fff_cost / $nat_cost) * 100" | bc 2>/dev/null || echo "?")
|
||||
echo " Cost savings: ${cost_savings}% (fff: \$$fff_cost_fmt, native: \$$nat_cost_fmt)"
|
||||
fi
|
||||
|
||||
echo " WINNER: $winner"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ─── MAIN ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
SELECTED=""
|
||||
MODE="both" # both, fff-only, native-only
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--fff-only) MODE="fff"; shift ;;
|
||||
--native-only) MODE="native"; shift ;;
|
||||
--model) MODEL="$2"; shift 2 ;;
|
||||
--max-turns) MAX_TURNS="$2"; shift 2 ;;
|
||||
--timeout) TIMEOUT_SEC="$2"; shift 2 ;;
|
||||
[0-9]*) SELECTED="$1"; shift ;;
|
||||
*)
|
||||
echo "Usage: $0 [1-10] [options]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --fff-only Only run fff MCP (skip native)"
|
||||
echo " --native-only Only run native tools (skip fff)"
|
||||
echo " --model MODEL Use specific model (e.g., haiku, sonnet)"
|
||||
echo " --max-turns N Max agentic turns per run (default: 10)"
|
||||
echo " --timeout SEC Timeout per run in seconds (default: 300)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "╔════════════════════════════════════════════════════════════════════════════╗"
|
||||
echo " Target: ~/dev/lightsource (194K files) "
|
||||
echo " Max turns: $MAX_TURNS | Timeout: ${TIMEOUT_SEC}s | Budget: \$0.50/run"
|
||||
echo "╚════════════════════════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
|
||||
if [[ -n "$SELECTED" ]]; then
|
||||
concepts=("$SELECTED")
|
||||
else
|
||||
concepts=(1 2 3 4 5 6 7 8 9 10 11)
|
||||
fi
|
||||
|
||||
for c in "${concepts[@]}"; do
|
||||
echo "── Concept $c: ${NAMES[$c]} ──"
|
||||
|
||||
if [[ "$MODE" == "both" || "$MODE" == "fff" ]]; then
|
||||
run_claude "fff" "$c"
|
||||
fi
|
||||
|
||||
if [[ "$MODE" == "both" || "$MODE" == "native" ]]; then
|
||||
run_claude "native" "$c"
|
||||
fi
|
||||
|
||||
if [[ "$MODE" == "both" ]]; then
|
||||
print_comparison "$c"
|
||||
fi
|
||||
done
|
||||
|
||||
# ─── FINAL ANALYSIS ──────────────────────────────────────────────────────────
|
||||
|
||||
if [[ "$MODE" == "both" && ${#concepts[@]} -ge 3 ]]; then
|
||||
echo ""
|
||||
echo "╔════════════════════════════════════════════════════════════════════════════╗"
|
||||
echo " ANALYSIS "
|
||||
echo "╚════════════════════════════════════════════════════════════════════════════╝"
|
||||
|
||||
python3 - "$RESULTS_DIR" "${concepts[*]}" <<'ANALYSIS_EOF'
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
|
||||
results_dir = sys.argv[1]
|
||||
concepts = [int(x) for x in sys.argv[2].split()]
|
||||
|
||||
NAMES = {
|
||||
1: "fuzzy-function-search",
|
||||
2: "api-endpoint-discovery",
|
||||
3: "cross-service-config",
|
||||
4: "test-file-discovery",
|
||||
5: "error-type-definition",
|
||||
6: "database-model-search",
|
||||
7: "auth-flow-tracing",
|
||||
8: "todo-tech-debt",
|
||||
9: "cross-language-pattern",
|
||||
10: "broad-pattern-search",
|
||||
11: "file-by-name-lookup",
|
||||
}
|
||||
|
||||
def load_result(name, mode):
|
||||
path = os.path.join(results_dir, f"{name}-{mode}.json")
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except:
|
||||
return None
|
||||
|
||||
def load_traces(name, mode):
|
||||
"""Extract tool calls with their input and result sizes from stream file."""
|
||||
path = os.path.join(results_dir, f"{name}-{mode}.stream.jsonl")
|
||||
if not os.path.exists(path):
|
||||
return []
|
||||
|
||||
tool_calls = []
|
||||
result_sizes = {}
|
||||
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(line)
|
||||
except:
|
||||
continue
|
||||
|
||||
content_blocks = []
|
||||
msg_type = msg.get("type", "")
|
||||
if msg_type == "assistant" and "message" in msg:
|
||||
content_blocks = msg["message"].get("content", [])
|
||||
elif msg_type == "user" and "message" in msg:
|
||||
content_blocks = msg["message"].get("content", [])
|
||||
|
||||
for block in content_blocks:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
if block.get("type") == "tool_use":
|
||||
inp = block.get("input", {})
|
||||
tool_calls.append({
|
||||
"name": block.get("name", "?"),
|
||||
"id": block.get("id", ""),
|
||||
"input": inp,
|
||||
"query": inp.get("query", inp.get("pattern", inp.get("patterns", inp.get("file_path", "")))),
|
||||
})
|
||||
if block.get("type") == "tool_result":
|
||||
tid = block.get("tool_use_id", "")
|
||||
content = block.get("content", "")
|
||||
if isinstance(content, list):
|
||||
total = sum(len(c.get("text", "")) for c in content if isinstance(c, dict))
|
||||
elif isinstance(content, str):
|
||||
total = len(content)
|
||||
else:
|
||||
total = len(str(content))
|
||||
result_sizes[tid] = total
|
||||
|
||||
for tc in tool_calls:
|
||||
tc["result_chars"] = result_sizes.get(tc["id"], -1)
|
||||
|
||||
return tool_calls
|
||||
|
||||
# ── Collect all data ──
|
||||
rows = []
|
||||
total_fff = 0
|
||||
total_nat = 0
|
||||
fff_wins = 0
|
||||
nat_wins = 0
|
||||
ties = 0
|
||||
|
||||
for c in concepts:
|
||||
name = NAMES.get(c, f"concept-{c}")
|
||||
fff = load_result(name, "fff")
|
||||
nat = load_result(name, "native")
|
||||
if not fff or not nat:
|
||||
continue
|
||||
|
||||
fc = fff.get("total_cost_usd", 0)
|
||||
nc = nat.get("total_cost_usd", 0)
|
||||
ft = fff.get("num_turns", 0)
|
||||
nt = nat.get("num_turns", 0)
|
||||
fw = fff.get("wall_ms", 0) / 1000
|
||||
nw = nat.get("wall_ms", 0) / 1000
|
||||
|
||||
total_fff += fc
|
||||
total_nat += nc
|
||||
|
||||
if nc > 0:
|
||||
ratio = fc / nc
|
||||
else:
|
||||
ratio = 1.0
|
||||
|
||||
if ratio < 0.85:
|
||||
winner = "FFF"
|
||||
fff_wins += 1
|
||||
elif ratio > 1.15:
|
||||
winner = "NATIVE"
|
||||
nat_wins += 1
|
||||
else:
|
||||
winner = "TIE"
|
||||
ties += 1
|
||||
|
||||
fff_traces = load_traces(name, "fff")
|
||||
nat_traces = load_traces(name, "native")
|
||||
|
||||
rows.append({
|
||||
"num": c, "name": name,
|
||||
"fff_cost": fc, "nat_cost": nc,
|
||||
"fff_turns": ft, "nat_turns": nt,
|
||||
"fff_wall": fw, "nat_wall": nw,
|
||||
"winner": winner, "ratio": ratio,
|
||||
"fff_traces": fff_traces, "nat_traces": nat_traces,
|
||||
})
|
||||
|
||||
# ── Summary table ──
|
||||
print()
|
||||
print(f" {'#':>2} {'Concept':<28} {'FFF $':>8} {'Nat $':>8} {'Δ':>6} {'FFF T':>5} {'Nat T':>5} {'Winner':>8}")
|
||||
print(f" {'─'*2} {'─'*28} {'─'*8} {'─'*8} {'─'*6} {'─'*5} {'─'*5} {'─'*8}")
|
||||
|
||||
for r in rows:
|
||||
savings = (1 - r["ratio"]) * 100
|
||||
print(f" {r['num']:>2} {r['name']:<28} ${r['fff_cost']:.4f} ${r['nat_cost']:.4f} {savings:>+5.0f}% {r['fff_turns']:>5} {r['nat_turns']:>5} {r['winner']:>8}")
|
||||
|
||||
if total_nat > 0:
|
||||
overall = (1 - total_fff / total_nat) * 100
|
||||
else:
|
||||
overall = 0
|
||||
|
||||
print()
|
||||
print(f" Score: FFF {fff_wins} | Native {nat_wins} | Tie {ties}")
|
||||
print(f" Total: FFF ${total_fff:.4f} | Native ${total_nat:.4f} | Savings: {overall:+.1f}%")
|
||||
print()
|
||||
|
||||
# ── Waste pattern analysis ──
|
||||
print(" ┌─ WASTE ANALYSIS ────────────────────────────────────────────────────")
|
||||
|
||||
for r in rows:
|
||||
traces = r["fff_traces"]
|
||||
if not traces:
|
||||
continue
|
||||
|
||||
issues = []
|
||||
|
||||
# Count tool types
|
||||
tool_search_calls = [t for t in traces if t["name"] == "ToolSearch"]
|
||||
read_calls = [t for t in traces if t["name"] == "Read"]
|
||||
grep_calls = [t for t in traces if "grep" in t["name"].lower()]
|
||||
find_calls = [t for t in traces if "find" in t["name"].lower()]
|
||||
|
||||
# Issue: ToolSearch overhead (each costs ~a turn)
|
||||
if len(tool_search_calls) >= 2:
|
||||
issues.append(f"{len(tool_search_calls)} ToolSearch calls (model loading tools in multiple turns)")
|
||||
|
||||
# Issue: Read after grep (grep didn't give enough context)
|
||||
if read_calls and grep_calls:
|
||||
read_files = set()
|
||||
for rc in read_calls:
|
||||
fp = rc.get("input", {}).get("file_path", "")
|
||||
if fp:
|
||||
read_files.add(fp.split("/")[-1])
|
||||
issues.append(f"Read calls after grep ({', '.join(read_files)}) — grep output wasn't sufficient")
|
||||
|
||||
# Issue: Many grep calls with tiny results (model is probing)
|
||||
tiny_greps = [t for t in grep_calls if 0 <= t["result_chars"] <= 50]
|
||||
if len(tiny_greps) >= 2:
|
||||
queries = [str(t["query"])[:40] for t in tiny_greps]
|
||||
issues.append(f"{len(tiny_greps)} greps returned ≤50 chars: {queries}")
|
||||
|
||||
# Issue: Large result from Read (could have been avoided)
|
||||
for rc in read_calls:
|
||||
if rc["result_chars"] > 5000:
|
||||
fn = rc.get("input", {}).get("file_path", "?").split("/")[-1]
|
||||
issues.append(f"Read({fn}) returned {rc['result_chars']:,} chars — expensive")
|
||||
|
||||
# Issue: grep returned huge result (over-broad query)
|
||||
for gc in grep_calls:
|
||||
if gc["result_chars"] > 3000:
|
||||
q = str(gc["query"])[:40]
|
||||
issues.append(f"Grep({q}) returned {gc['result_chars']:,} chars — too broad")
|
||||
|
||||
# Issue: Sequential greps that could have been multi_grep
|
||||
if len(grep_calls) >= 3 and not any("multi" in t["name"].lower() for t in traces):
|
||||
issues.append(f"{len(grep_calls)} sequential greps — could multi_grep reduce to 1 call?")
|
||||
|
||||
if issues and r["winner"] != "FFF":
|
||||
print(f" │")
|
||||
savings = (1 - r["ratio"]) * 100
|
||||
print(f" │ #{r['num']} {r['name']} ({r['winner']}, {savings:+.0f}%)")
|
||||
for issue in issues:
|
||||
print(f" │ • {issue}")
|
||||
|
||||
# Show fff trace summary
|
||||
trace_summary = " → ".join(
|
||||
t["name"].replace("mcp__fff__", "").replace("ToolSearch", "🔍")
|
||||
for t in traces
|
||||
)
|
||||
print(f" │ trace: {trace_summary}")
|
||||
|
||||
# Show native trace for comparison
|
||||
nat_traces = r["nat_traces"]
|
||||
if nat_traces:
|
||||
nat_summary = " → ".join(
|
||||
t["name"].replace("ToolSearch", "🔍")
|
||||
for t in nat_traces
|
||||
)
|
||||
print(f" │ native: {nat_summary}")
|
||||
|
||||
print(" │")
|
||||
print(" └────────────────────────────────────────────────────────────────────")
|
||||
print()
|
||||
|
||||
# ── Actionable suggestions ──
|
||||
print(" ┌─ SUGGESTED IMPROVEMENTS ────────────────────────────────────────────")
|
||||
|
||||
# Aggregate patterns across all concepts
|
||||
total_tool_search = sum(len([t for t in r["fff_traces"] if t["name"] == "ToolSearch"]) for r in rows)
|
||||
total_reads_after_grep = sum(
|
||||
1 for r in rows
|
||||
if any("grep" in t["name"].lower() for t in r["fff_traces"])
|
||||
and any(t["name"] == "Read" for t in r["fff_traces"])
|
||||
)
|
||||
total_tiny_greps = sum(
|
||||
len([t for t in r["fff_traces"] if "grep" in t["name"].lower() and 0 <= t["result_chars"] <= 50])
|
||||
for r in rows
|
||||
)
|
||||
total_sequential_greps = sum(
|
||||
len([t for t in r["fff_traces"] if "grep" in t["name"].lower()])
|
||||
for r in rows
|
||||
if len([t for t in r["fff_traces"] if "grep" in t["name"].lower()]) >= 3
|
||||
)
|
||||
|
||||
if total_reads_after_grep >= 3:
|
||||
print(f" │ 1. EXPAND GREP CONTEXT: {total_reads_after_grep}/{len(rows)} concepts do Read after grep.")
|
||||
print(f" │ Grep results need more inline context to avoid follow-up Reads.")
|
||||
print(f" │ → Increase MAX_DEF_EXPAND, show more body for non-def matches")
|
||||
print(f" │")
|
||||
|
||||
if total_tiny_greps >= 5:
|
||||
print(f" │ 2. IMPROVE ZERO/LOW-RESULT GUIDANCE: {total_tiny_greps} greps returned ≤50 chars.")
|
||||
print(f" │ When results are sparse, show related files/symbols to help the model.")
|
||||
print(f" │ → Add 'did you mean?' suggestions or sibling files in same directory")
|
||||
print(f" │")
|
||||
|
||||
if total_tool_search >= len(rows) * 1.5:
|
||||
print(f" │ 3. REDUCE TOOLSEARCH OVERHEAD: {total_tool_search} ToolSearch calls across {len(rows)} concepts.")
|
||||
print(f" │ Each ToolSearch costs a turn. Model loads tools incrementally.")
|
||||
print(f" │ → Can't fix directly, but reducing total calls makes this less impactful")
|
||||
print(f" │")
|
||||
|
||||
if total_sequential_greps >= 8:
|
||||
print(f" │ 4. PROMOTE MULTI_GREP: {total_sequential_greps} sequential grep calls could be batched.")
|
||||
print(f" │ Model uses sequential grep when multi_grep would be more efficient.")
|
||||
print(f" │ → Improve multi_grep description or auto-suggest in 0-result messages")
|
||||
print(f" │")
|
||||
|
||||
# Concept-specific suggestions
|
||||
losing = [r for r in rows if r["winner"] == "NATIVE"]
|
||||
if losing:
|
||||
print(f" │ LOSING CONCEPTS ({len(losing)}):")
|
||||
for r in losing:
|
||||
traces = r["fff_traces"]
|
||||
nat_traces = r["nat_traces"]
|
||||
fff_grep_count = len([t for t in traces if "grep" in t["name"].lower()])
|
||||
nat_grep_count = len([t for t in nat_traces if "grep" in t["name"].lower() or t["name"] == "Grep"])
|
||||
fff_read_count = len([t for t in traces if t["name"] == "Read"])
|
||||
nat_read_count = len([t for t in nat_traces if t["name"] == "Read"])
|
||||
|
||||
savings = (1 - r["ratio"]) * 100
|
||||
print(f" │ #{r['num']} {r['name']} ({savings:+.0f}%): fff={fff_grep_count}grep+{fff_read_count}read vs nat={nat_grep_count}grep+{nat_read_count}read")
|
||||
|
||||
print(" │")
|
||||
print(" └────────────────────────────────────────────────────────────────────")
|
||||
print()
|
||||
ANALYSIS_EOF
|
||||
fi
|
||||
@@ -0,0 +1,32 @@
|
||||
local script_path = arg[0]
|
||||
local script_dir = script_path:match('(.*[/\\])') or './'
|
||||
local repo_root = script_dir .. '..'
|
||||
|
||||
-- Add the plugin's lua/ directory to the module search path
|
||||
package.path = repo_root .. '/lua/?.lua;' .. repo_root .. '/lua/?/init.lua;' .. package.path
|
||||
|
||||
local version = require('fff.utils.version')
|
||||
|
||||
local info, err = version.resolve(repo_root)
|
||||
if not info then
|
||||
io.stderr:write('Error: ' .. (err or 'unknown') .. '\n')
|
||||
os.exit(1)
|
||||
end
|
||||
|
||||
print('version=' .. info.version)
|
||||
print('release_tag=' .. info.release_tag)
|
||||
print('npm_tag=' .. info.npm_tag)
|
||||
print('is_release=' .. tostring(info.is_release))
|
||||
|
||||
-- Write to GITHUB_OUTPUT when running in CI
|
||||
local github_output = os.getenv('GITHUB_OUTPUT')
|
||||
if github_output and github_output ~= '' then
|
||||
local f = io.open(github_output, 'a')
|
||||
if f then
|
||||
f:write('version=' .. info.version .. '\n')
|
||||
f:write('release_tag=' .. info.release_tag .. '\n')
|
||||
f:write('npm_tag=' .. info.npm_tag .. '\n')
|
||||
f:write('is_release=' .. tostring(info.is_release) .. '\n')
|
||||
f:close()
|
||||
end
|
||||
end
|
||||
Executable
+233
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Diagnose LMDB lock.mdb holders and waiters.
|
||||
|
||||
Usage: ./diagnose_lockmdb.py [path_to_lock.mdb]
|
||||
Default: /home/neogoose/.cache/nvim/fff_nvim/lock.mdb
|
||||
|
||||
Shows:
|
||||
1. Every process that has the lock.mdb inode in its mmap/fd table (potential holders)
|
||||
2. Every process currently blocked on a futex at an address in the lock.mdb mmap
|
||||
(actually waiting on the reader/writer mutex)
|
||||
3. The mutex state (__lock / __owner / __kind) for both reader and writer mutex
|
||||
"""
|
||||
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
LOCK_PATH = sys.argv[1] if len(sys.argv) > 1 else "/home/neogoose/.cache/nvim/fff_nvim/lock.mdb"
|
||||
|
||||
RMUTEX_OFFSET = 24 # MDB_txbody.mtb_rmutex
|
||||
WMUTEX_OFFSET = 64 # MDB_txninfo.mt2.mt2_wmutex
|
||||
MUTEX_SIZE = 40 # sizeof(pthread_mutex_t) on glibc x86_64
|
||||
|
||||
|
||||
def mutex_state(data, off):
|
||||
"""Decode pthread_mutex_t at offset `off`."""
|
||||
if len(data) < off + 20:
|
||||
return None
|
||||
return {
|
||||
"lock": struct.unpack("i", data[off:off + 4])[0],
|
||||
"count": struct.unpack("I", data[off + 4:off + 8])[0],
|
||||
"owner": struct.unpack("i", data[off + 8:off + 12])[0],
|
||||
"nusers": struct.unpack("I", data[off + 12:off + 16])[0],
|
||||
"kind": struct.unpack("i", data[off + 16:off + 20])[0],
|
||||
}
|
||||
|
||||
|
||||
def fmt_mutex(m):
|
||||
if m is None:
|
||||
return "(too small)"
|
||||
robust = "ROBUST" if m["kind"] & 0x10 else "non-robust"
|
||||
pshared = "PSHARED" if m["kind"] & 0x80 else "private"
|
||||
state = {0: "unlocked", 1: "locked", 2: "locked+waiters"}.get(m["lock"], f"lock={m['lock']}")
|
||||
return (f"state={state} owner_tid={m['owner']} nusers={m['nusers']} "
|
||||
f"kind=0x{m['kind']:x} ({robust}, {pshared})")
|
||||
|
||||
|
||||
def get_comm(pid):
|
||||
try:
|
||||
return Path(f"/proc/{pid}/comm").read_text().strip()
|
||||
except Exception:
|
||||
return "?"
|
||||
|
||||
|
||||
def get_cmdline(pid):
|
||||
try:
|
||||
return Path(f"/proc/{pid}/cmdline").read_text().replace("\0", " ").strip()[:100]
|
||||
except Exception:
|
||||
return "?"
|
||||
|
||||
|
||||
def get_wchan(pid_or_tid, path):
|
||||
try:
|
||||
return Path(f"{path}/wchan").read_text().strip()
|
||||
except Exception:
|
||||
return "?"
|
||||
|
||||
|
||||
def get_syscall(tid_path):
|
||||
"""Return (syscall_nr, arg0, arg1, ..., arg5) or None."""
|
||||
try:
|
||||
parts = Path(f"{tid_path}/syscall").read_text().split()
|
||||
if parts and parts[0] != "running":
|
||||
return [int(p, 0) for p in parts[:7]]
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def get_mmap_regions(pid, target_inode):
|
||||
"""Return list of (start, end) virtual addresses where `target_inode` is mapped."""
|
||||
regions = []
|
||||
try:
|
||||
for line in Path(f"/proc/{pid}/maps").read_text().splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
# inode is field index 4
|
||||
try:
|
||||
inode = int(parts[4])
|
||||
except ValueError:
|
||||
continue
|
||||
if inode != target_inode:
|
||||
continue
|
||||
addr_range = parts[0]
|
||||
try:
|
||||
start_s, end_s = addr_range.split("-")
|
||||
regions.append((int(start_s, 16), int(end_s, 16), line))
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
return regions
|
||||
|
||||
|
||||
def main():
|
||||
path = Path(LOCK_PATH)
|
||||
if not path.exists():
|
||||
print(f"ERROR: {path} does not exist")
|
||||
return 1
|
||||
|
||||
st = path.stat()
|
||||
target_inode = st.st_ino
|
||||
size = st.st_size
|
||||
print(f"=== Target: {path} ===")
|
||||
print(f" inode={target_inode} size={size}\n")
|
||||
|
||||
# Read mutex state from the on-disk file
|
||||
data = path.read_bytes()
|
||||
r_mutex = mutex_state(data, RMUTEX_OFFSET)
|
||||
w_mutex = mutex_state(data, WMUTEX_OFFSET)
|
||||
print("=== Mutex state on disk ===")
|
||||
print(f" READER mutex (offset {RMUTEX_OFFSET}): {fmt_mutex(r_mutex)}")
|
||||
print(f" WRITER mutex (offset {WMUTEX_OFFSET}): {fmt_mutex(w_mutex)}")
|
||||
print()
|
||||
|
||||
# Scan all processes for mmap hits on this inode.
|
||||
holders = [] # list of (pid, start, end, deleted)
|
||||
for pid_dir in Path("/proc").iterdir():
|
||||
if not pid_dir.name.isdigit():
|
||||
continue
|
||||
pid = int(pid_dir.name)
|
||||
regions = get_mmap_regions(pid, target_inode)
|
||||
if not regions:
|
||||
# Also check fd table — inode may be open-but-not-mmap'd
|
||||
try:
|
||||
for fd in (pid_dir / "fd").iterdir():
|
||||
try:
|
||||
tgt = os.readlink(fd)
|
||||
except OSError:
|
||||
continue
|
||||
if "lock.mdb" in tgt and f"[{target_inode}]" in os.stat(fd).st_dev.__str__():
|
||||
# crude — we already caught mmap above, skip
|
||||
pass
|
||||
except OSError:
|
||||
pass
|
||||
continue
|
||||
for (start, end, line) in regions:
|
||||
deleted = "(deleted)" in line
|
||||
holders.append((pid, start, end, deleted))
|
||||
|
||||
print("=== Processes with lock.mdb mmap'd ===")
|
||||
if not holders:
|
||||
print(" (none — no live process holds this inode via mmap)")
|
||||
for (pid, start, end, deleted) in holders:
|
||||
marker = " (DELETED inode — stale mapping from previous file)" if deleted else ""
|
||||
print(f" PID {pid} ({get_comm(pid)}) mmap @ 0x{start:x}-0x{end:x}{marker}")
|
||||
print(f" cmdline: {get_cmdline(pid)}")
|
||||
|
||||
print()
|
||||
|
||||
# For each holder mmap, compute the futex addresses of each mutex,
|
||||
# then scan all threads in all processes for futex syscalls targeting those addresses.
|
||||
futex_addrs = {} # address -> description
|
||||
for (pid, start, _, deleted) in holders:
|
||||
futex_addrs[start + RMUTEX_OFFSET] = (pid, "READER mutex", deleted)
|
||||
futex_addrs[start + WMUTEX_OFFSET] = (pid, "WRITER mutex", deleted)
|
||||
|
||||
print("=== Threads currently blocked on mutex futex ===")
|
||||
found_any = False
|
||||
for pid_dir in Path("/proc").iterdir():
|
||||
if not pid_dir.name.isdigit():
|
||||
continue
|
||||
pid = int(pid_dir.name)
|
||||
task_dir = pid_dir / "task"
|
||||
if not task_dir.exists():
|
||||
continue
|
||||
try:
|
||||
tids = [t.name for t in task_dir.iterdir() if t.name.isdigit()]
|
||||
except OSError:
|
||||
continue
|
||||
for tid in tids:
|
||||
tid_path = task_dir / tid
|
||||
sc = get_syscall(tid_path)
|
||||
if sc is None:
|
||||
continue
|
||||
# syscall 202 = futex on x86_64
|
||||
if sc[0] != 202:
|
||||
continue
|
||||
futex_addr = sc[1]
|
||||
# Match: the blocked thread must have this address mapped into its own address space.
|
||||
# Since mmap addresses differ per process, compare only against THIS process's own holders.
|
||||
my_regions = get_mmap_regions(pid, target_inode)
|
||||
for (start, end, line) in my_regions:
|
||||
deleted = "(deleted)" in line
|
||||
r_addr = start + RMUTEX_OFFSET
|
||||
w_addr = start + WMUTEX_OFFSET
|
||||
if futex_addr == r_addr:
|
||||
wchan = get_wchan(tid, tid_path)
|
||||
dmark = " (DELETED)" if deleted else ""
|
||||
print(f" PID {pid} TID {tid} ({get_comm(pid)}) wchan={wchan}: "
|
||||
f"blocked on READER mutex @ 0x{futex_addr:x}{dmark}")
|
||||
found_any = True
|
||||
elif futex_addr == w_addr:
|
||||
wchan = get_wchan(tid, tid_path)
|
||||
dmark = " (DELETED)" if deleted else ""
|
||||
print(f" PID {pid} TID {tid} ({get_comm(pid)}) wchan={wchan}: "
|
||||
f"blocked on WRITER mutex @ 0x{futex_addr:x}{dmark}")
|
||||
found_any = True
|
||||
if not found_any:
|
||||
print(" (none)")
|
||||
|
||||
print()
|
||||
print("=== /proc/locks fcntl/flock state ===")
|
||||
try:
|
||||
for line in Path("/proc/locks").read_text().splitlines():
|
||||
if str(target_inode) in line:
|
||||
# format: N: TYPE ACCESS KIND PID MAJ:MIN:INODE START END
|
||||
parts = line.split()
|
||||
if len(parts) >= 7 and parts[6].endswith(f":{target_inode}"):
|
||||
pid = parts[4]
|
||||
print(f" {line}")
|
||||
print(f" -> PID {pid} ({get_comm(pid)})")
|
||||
except Exception as e:
|
||||
print(f" error reading /proc/locks: {e}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="${1:?Usage: ./scripts/release.sh <version> (e.g. 0.3.0)}"
|
||||
|
||||
# Strip leading 'v' if provided
|
||||
VERSION="${VERSION#v}"
|
||||
|
||||
TAG="v${VERSION}"
|
||||
|
||||
git pull
|
||||
|
||||
# Check for clean working tree
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
echo "Error: Working tree is not clean. Commit or stash changes first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if tag already exists
|
||||
if git rev-parse "$TAG" >/dev/null 2>&1; then
|
||||
echo "Error: Tag $TAG already exists"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Releasing $VERSION..."
|
||||
|
||||
echo "→ Ensuring cargo-edit is installed (force avoids a shadowing cargo-set-version binary)"
|
||||
cargo install cargo-edit --force --locked
|
||||
|
||||
echo "→ Updating Cargo.toml versions to $VERSION"
|
||||
cargo set-version "$VERSION"
|
||||
|
||||
echo "→ Updating Python package version to $VERSION"
|
||||
PYPROJECT="packages/fff-python/pyproject.toml"
|
||||
PYINIT="packages/fff-python/src/fff/__init__.py"
|
||||
UVLOCK="packages/fff-python/uv.lock"
|
||||
|
||||
sed_inplace() {
|
||||
local file="$1"; shift
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
sed "$@" "$file" >"$tmp"
|
||||
mv "$tmp" "$file"
|
||||
}
|
||||
|
||||
sed_inplace "$PYPROJECT" -e 's/^version = ".*"$/version = "'"$VERSION"'"/'
|
||||
sed_inplace "$PYINIT" -e 's/^__version__ = ".*"$/__version__ = "'"$VERSION"'"/'
|
||||
|
||||
# uv.lock has a `version` line per package; only touch the one directly
|
||||
# after the `fff-search` package entry. Multi `-e` keeps the `{ }` block
|
||||
# portable across BSD and GNU sed.
|
||||
if [ -f "$UVLOCK" ]; then
|
||||
sed_inplace "$UVLOCK" \
|
||||
-e '/^name = "fff-search"$/{' \
|
||||
-e 'n' \
|
||||
-e 's/^version = ".*"$/version = "'"$VERSION"'"/' \
|
||||
-e '}'
|
||||
fi
|
||||
|
||||
# Fail loudly if any substitution did not land (mirrors the old guard).
|
||||
grep -q "^version = \"$VERSION\"$" "$PYPROJECT" || { echo "Error: failed to update version in $PYPROJECT"; exit 1; }
|
||||
grep -q "^__version__ = \"$VERSION\"$" "$PYINIT" || { echo "Error: failed to update version in $PYINIT"; exit 1; }
|
||||
if [ -f "$UVLOCK" ]; then
|
||||
grep -A1 '^name = "fff-search"$' "$UVLOCK" | grep -q "^version = \"$VERSION\"$" \
|
||||
|| { echo "Error: failed to update fff-search version in $UVLOCK"; exit 1; }
|
||||
fi
|
||||
|
||||
git add -A
|
||||
git commit -m "chore: release $VERSION"
|
||||
|
||||
echo "→ Creating tag $TAG"
|
||||
git tag -a "$TAG" -m "Release $VERSION"
|
||||
|
||||
echo "→ Pushing to origin"
|
||||
git push origin
|
||||
git push origin "$TAG"
|
||||
|
||||
echo ""
|
||||
echo "Release $VERSION created and pushed."
|
||||
echo "CI will build and publish: https://github.com/dmtrKovalenko/fff.nvim/actions"
|
||||
Reference in New Issue
Block a user