chore: import upstream snapshot with attribution

scripts/compare_token_usage.py

@@ -0,0 +1,500 @@
+#!/usr/bin/env python3
+"""
+Compare token usage between jcode and Claude Code CLI.
+
+This script runs the same prompts through both tools and compares their token usage.
+The goal is to verify that jcode's token consumption is within expected bounds
+compared to the official Claude Code CLI.
+
+NOTE: jcode typically uses FEWER tokens than Claude CLI because:
+1. jcode has a smaller/simpler system prompt
+2. jcode registers fewer tools (Claude CLI has many built-in tools)
+3. Different prompt caching behavior
+
+The test PASSES if jcode uses fewer tokens OR at most 50% more tokens.
+Using more tokens would indicate a problem with the system prompt or tool registration.
+
+Usage:
+    python scripts/compare_token_usage.py [--verbose] [--runs N]
+
+Requirements:
+    - jcode built and in PATH or at target/release/jcode
+    - claude CLI installed and authenticated
+    - Both should use the same model (claude-opus-4-5-20251101 by default)
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+
+@dataclass
+class TokenUsage:
+    """Token usage from a single run."""
+    input_tokens: int
+    output_tokens: int
+    cache_read_tokens: int
+    cache_creation_tokens: int
+    total_cost_usd: Optional[float] = None
+    duration_ms: Optional[int] = None
+
+    @property
+    def total_input(self) -> int:
+        """Total input tokens including cache."""
+        return self.input_tokens + self.cache_read_tokens + self.cache_creation_tokens
+
+    @property
+    def total(self) -> int:
+        """Total tokens (input + output)."""
+        return self.total_input + self.output_tokens
+
+
+@dataclass
+class RunResult:
+    """Result of a single tool run."""
+    tool: str
+    prompt: str
+    usage: TokenUsage
+    success: bool
+    output: str
+    error: Optional[str] = None
+
+
+def find_jcode_binary() -> str:
+    """Find the jcode binary."""
+    # Check target/release first
+    repo_root = Path(__file__).parent.parent
+    release_binary = repo_root / "target" / "release" / "jcode"
+    if release_binary.exists():
+        return str(release_binary)
+
+    # Check PATH
+    result = subprocess.run(["which", "jcode"], capture_output=True, text=True)
+    if result.returncode == 0:
+        return result.stdout.strip()
+
+    raise FileNotFoundError("jcode binary not found. Run 'cargo build --release' first.")
+
+
+def run_claude_cli(prompt: str, workdir: str, model: str = "opus") -> RunResult:
+    """Run the Claude Code CLI and capture token usage."""
+    try:
+        result = subprocess.run(
+            [
+                "claude",
+                "-p",
+                "--output-format", "json",
+                "--dangerously-skip-permissions",
+                "--model", model,
+                prompt,
+            ],
+            capture_output=True,
+            text=True,
+            cwd=workdir,
+            timeout=120,
+        )
+
+        if result.returncode != 0 and not result.stdout:
+            return RunResult(
+                tool="claude",
+                prompt=prompt,
+                usage=TokenUsage(0, 0, 0, 0),
+                success=False,
+                output="",
+                error=result.stderr or f"Exit code {result.returncode}",
+            )
+
+        # Parse JSON output
+        data = json.loads(result.stdout)
+        usage = data.get("usage", {})
+
+        token_usage = TokenUsage(
+            input_tokens=usage.get("input_tokens", 0),
+            output_tokens=usage.get("output_tokens", 0),
+            cache_read_tokens=usage.get("cache_read_input_tokens", 0),
+            cache_creation_tokens=usage.get("cache_creation_input_tokens", 0),
+            total_cost_usd=data.get("total_cost_usd"),
+            duration_ms=data.get("duration_ms"),
+        )
+
+        return RunResult(
+            tool="claude",
+            prompt=prompt,
+            usage=token_usage,
+            success=not data.get("is_error", False),
+            output=data.get("result", ""),
+        )
+
+    except subprocess.TimeoutExpired:
+        return RunResult(
+            tool="claude",
+            prompt=prompt,
+            usage=TokenUsage(0, 0, 0, 0),
+            success=False,
+            output="",
+            error="Timeout after 120s",
+        )
+    except json.JSONDecodeError as e:
+        return RunResult(
+            tool="claude",
+            prompt=prompt,
+            usage=TokenUsage(0, 0, 0, 0),
+            success=False,
+            output=result.stdout if 'result' in dir() else "",
+            error=f"JSON parse error: {e}",
+        )
+    except Exception as e:
+        return RunResult(
+            tool="claude",
+            prompt=prompt,
+            usage=TokenUsage(0, 0, 0, 0),
+            success=False,
+            output="",
+            error=str(e),
+        )
+
+
+def run_jcode(prompt: str, workdir: str, jcode_binary: str, model: str = "claude-opus-4-5-20251101") -> RunResult:
+    """Run jcode and capture token usage from trace output."""
+    try:
+        # Create a temporary JCODE_HOME to avoid polluting user's sessions
+        with tempfile.TemporaryDirectory() as tmpdir:
+            env = os.environ.copy()
+            env["JCODE_HOME"] = tmpdir
+            env["JCODE_TRACE"] = "1"
+
+            result = subprocess.run(
+                [
+                    jcode_binary,
+                    "run",
+                    "--no-update",
+                    "--model", model,
+                    prompt,
+                ],
+                capture_output=True,
+                text=True,
+                cwd=workdir,
+                timeout=120,
+                env=env,
+            )
+
+            # Parse token usage from trace output in stderr
+            # Format: [trace] token_usage input=X output=Y cache_read=Z cache_write=W
+            input_tokens = 0
+            output_tokens = 0
+            cache_read = 0
+            cache_write = 0
+
+            for line in result.stderr.split("\n"):
+                if "[trace] token_usage" in line:
+                    parts = line.split()
+                    for part in parts:
+                        if part.startswith("input="):
+                            input_tokens = int(part.split("=")[1])
+                        elif part.startswith("output="):
+                            output_tokens = int(part.split("=")[1])
+                        elif part.startswith("cache_read="):
+                            cache_read = int(part.split("=")[1])
+                        elif part.startswith("cache_write="):
+                            cache_write = int(part.split("=")[1])
+
+            token_usage = TokenUsage(
+                input_tokens=input_tokens,
+                output_tokens=output_tokens,
+                cache_read_tokens=cache_read,
+                cache_creation_tokens=cache_write,
+            )
+
+            return RunResult(
+                tool="jcode",
+                prompt=prompt,
+                usage=token_usage,
+                success=result.returncode == 0,
+                output=result.stdout,
+                error=None if result.returncode == 0 else result.stderr,
+            )
+
+    except subprocess.TimeoutExpired:
+        return RunResult(
+            tool="jcode",
+            prompt=prompt,
+            usage=TokenUsage(0, 0, 0, 0),
+            success=False,
+            output="",
+            error="Timeout after 120s",
+        )
+    except Exception as e:
+        return RunResult(
+            tool="jcode",
+            prompt=prompt,
+            usage=TokenUsage(0, 0, 0, 0),
+            success=False,
+            output="",
+            error=str(e),
+        )
+
+
+def compare_usage(claude_result: RunResult, jcode_result: RunResult, verbose: bool = False) -> dict:
+    """Compare token usage between Claude CLI and jcode."""
+    c = claude_result.usage
+    j = jcode_result.usage
+
+    # Calculate differences
+    input_diff = j.input_tokens - c.input_tokens
+    output_diff = j.output_tokens - c.output_tokens
+    cache_read_diff = j.cache_read_tokens - c.cache_read_tokens
+    cache_write_diff = j.cache_creation_tokens - c.cache_creation_tokens
+    total_diff = j.total - c.total
+
+    # Calculate percentages (avoid division by zero)
+    def pct_diff(a: int, b: int) -> float:
+        if b == 0:
+            return 0.0 if a == 0 else float('inf')
+        return ((a - b) / b) * 100
+
+    input_pct = pct_diff(j.input_tokens, c.input_tokens)
+    output_pct = pct_diff(j.output_tokens, c.output_tokens)
+    total_pct = pct_diff(j.total, c.total)
+
+    return {
+        "claude": {
+            "input": c.input_tokens,
+            "output": c.output_tokens,
+            "cache_read": c.cache_read_tokens,
+            "cache_write": c.cache_creation_tokens,
+            "total": c.total,
+            "cost_usd": c.total_cost_usd,
+            "duration_ms": c.duration_ms,
+        },
+        "jcode": {
+            "input": j.input_tokens,
+            "output": j.output_tokens,
+            "cache_read": j.cache_read_tokens,
+            "cache_write": j.cache_creation_tokens,
+            "total": j.total,
+        },
+        "diff": {
+            "input": input_diff,
+            "output": output_diff,
+            "cache_read": cache_read_diff,
+            "cache_write": cache_write_diff,
+            "total": total_diff,
+        },
+        "pct_diff": {
+            "input": input_pct,
+            "output": output_pct,
+            "total": total_pct,
+        },
+    }
+
+
+def print_comparison(comparison: dict, prompt: str, verbose: bool = False):
+    """Print a formatted comparison."""
+    print(f"\n{'='*60}")
+    print(f"Prompt: {prompt[:50]}..." if len(prompt) > 50 else f"Prompt: {prompt}")
+    print(f"{'='*60}")
+
+    c = comparison["claude"]
+    j = comparison["jcode"]
+    d = comparison["diff"]
+    p = comparison["pct_diff"]
+
+    print(f"\n{'Metric':<20} {'Claude':<15} {'jcode':<15} {'Diff':<15} {'% Diff':<10}")
+    print("-" * 75)
+    print(f"{'Input tokens':<20} {c['input']:<15} {j['input']:<15} {d['input']:+<15} {p['input']:+.1f}%")
+    print(f"{'Output tokens':<20} {c['output']:<15} {j['output']:<15} {d['output']:+<15} {p['output']:+.1f}%")
+    print(f"{'Cache read':<20} {c['cache_read']:<15} {j['cache_read']:<15} {d['cache_read']:+<15}")
+    print(f"{'Cache write':<20} {c['cache_write']:<15} {j['cache_write']:<15} {d['cache_write']:+<15}")
+    print("-" * 75)
+    print(f"{'TOTAL':<20} {c['total']:<15} {j['total']:<15} {d['total']:+<15} {p['total']:+.1f}%")
+
+    if c.get("cost_usd"):
+        print(f"\nClaude CLI cost: ${c['cost_usd']:.6f}")
+    if c.get("duration_ms"):
+        print(f"Claude CLI duration: {c['duration_ms']}ms")
+
+
+def run_test_suite(verbose: bool = False, runs: int = 1) -> list:
+    """Run the full test suite."""
+    # Test prompts - simple ones that don't require tools
+    prompts = [
+        "Reply with a single word: test",
+        "What is 2 + 2? Reply with just the number.",
+        "List three primary colors, one per line.",
+    ]
+
+    jcode_binary = find_jcode_binary()
+    print(f"Using jcode binary: {jcode_binary}")
+    print(f"Running {len(prompts)} prompts, {runs} run(s) each\n")
+
+    results = []
+
+    with tempfile.TemporaryDirectory() as workdir:
+        for prompt in prompts:
+            for run_num in range(runs):
+                print(f"\n[Run {run_num + 1}/{runs}] Testing: {prompt[:40]}...")
+
+                # Run both tools
+                print("  Running Claude CLI...", end=" ", flush=True)
+                claude_result = run_claude_cli(prompt, workdir)
+                if claude_result.success:
+                    print(f"OK ({claude_result.usage.total} tokens)")
+                else:
+                    print(f"FAILED: {claude_result.error}")
+                    if verbose:
+                        print(f"    Output: {claude_result.output[:200]}")
+
+                # Small delay to avoid rate limiting
+                time.sleep(1)
+
+                print("  Running jcode...", end=" ", flush=True)
+                jcode_result = run_jcode(prompt, workdir, jcode_binary)
+                if jcode_result.success:
+                    print(f"OK ({jcode_result.usage.total} tokens)")
+                else:
+                    print(f"FAILED: {jcode_result.error}")
+                    if verbose:
+                        print(f"    Output: {jcode_result.output[:200]}")
+
+                if claude_result.success and jcode_result.success:
+                    comparison = compare_usage(claude_result, jcode_result, verbose)
+                    results.append({
+                        "prompt": prompt,
+                        "run": run_num + 1,
+                        "comparison": comparison,
+                    })
+
+                    if verbose:
+                        print_comparison(comparison, prompt, verbose)
+
+                # Delay between prompts
+                time.sleep(2)
+
+    return results
+
+
+def summarize_results(results: list) -> bool:
+    """Print summary of all results. Returns True if test passed."""
+    if not results:
+        print("\nNo successful results to summarize.")
+        return False
+
+    print(f"\n{'='*60}")
+    print("SUMMARY")
+    print(f"{'='*60}")
+
+    total_claude = sum(r["comparison"]["claude"]["total"] for r in results)
+    total_jcode = sum(r["comparison"]["jcode"]["total"] for r in results)
+    total_diff = total_jcode - total_claude
+
+    # Also compare just input+output (excluding cache)
+    total_claude_io = sum(
+        r["comparison"]["claude"]["input"] + r["comparison"]["claude"]["output"]
+        for r in results
+    )
+    total_jcode_io = sum(
+        r["comparison"]["jcode"]["input"] + r["comparison"]["jcode"]["output"]
+        for r in results
+    )
+
+    if total_claude > 0:
+        pct_diff = ((total_jcode - total_claude) / total_claude) * 100
+    else:
+        pct_diff = 0
+
+    print(f"\nTotal runs: {len(results)}")
+    print(f"\n--- Total Tokens (including cache) ---")
+    print(f"Claude CLI: {total_claude}")
+    print(f"jcode:      {total_jcode}")
+    print(f"Difference: {total_diff:+} ({pct_diff:+.1f}%)")
+
+    print(f"\n--- Input + Output only (excluding cache) ---")
+    print(f"Claude CLI: {total_claude_io}")
+    print(f"jcode:      {total_jcode_io}")
+    if total_claude_io > 0:
+        io_pct_diff = ((total_jcode_io - total_claude_io) / total_claude_io) * 100
+        print(f"Difference: {total_jcode_io - total_claude_io:+} ({io_pct_diff:+.1f}%)")
+
+    # Check if within acceptable bounds
+    # jcode using fewer tokens is always good (negative diff)
+    # jcode using more tokens is acceptable up to MAX_OVERHEAD_PCT
+    MAX_OVERHEAD_PCT = 50  # Allow up to 50% more tokens (for different system prompts)
+
+    passed = True
+    if pct_diff <= 0:
+        print(f"\n✅ PASS: jcode uses {abs(pct_diff):.1f}% fewer tokens than Claude CLI")
+    elif pct_diff <= MAX_OVERHEAD_PCT:
+        print(f"\n✅ PASS: jcode uses {pct_diff:.1f}% more tokens (within {MAX_OVERHEAD_PCT}% threshold)")
+    else:
+        print(f"\n❌ FAIL: jcode uses {pct_diff:.1f}% more tokens (exceeds {MAX_OVERHEAD_PCT}% threshold)")
+        passed = False
+
+    # Per-prompt breakdown
+    print("\nPer-prompt breakdown:")
+    print(f"{'Prompt':<40} {'Claude':<10} {'jcode':<10} {'Diff':<10}")
+    print("-" * 70)
+
+    for r in results:
+        prompt = r["prompt"][:37] + "..." if len(r["prompt"]) > 40 else r["prompt"]
+        c_total = r["comparison"]["claude"]["total"]
+        j_total = r["comparison"]["jcode"]["total"]
+        diff = j_total - c_total
+        print(f"{prompt:<40} {c_total:<10} {j_total:<10} {diff:+<10}")
+
+    return passed
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Compare token usage between jcode and Claude Code CLI"
+    )
+    parser.add_argument(
+        "--verbose", "-v",
+        action="store_true",
+        help="Show detailed output for each run",
+    )
+    parser.add_argument(
+        "--runs", "-n",
+        type=int,
+        default=1,
+        help="Number of runs per prompt (default: 1)",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output results as JSON",
+    )
+
+    args = parser.parse_args()
+
+    print("Token Usage Comparison: jcode vs Claude Code CLI")
+    print("=" * 50)
+
+    try:
+        results = run_test_suite(verbose=args.verbose, runs=args.runs)
+
+        if args.json:
+            print(json.dumps(results, indent=2))
+            # For JSON mode, check if all runs succeeded
+            sys.exit(0 if results else 1)
+        else:
+            passed = summarize_results(results)
+            sys.exit(0 if passed else 1)
+
+    except FileNotFoundError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+    except KeyboardInterrupt:
+        print("\nInterrupted by user")
+        sys.exit(130)
+
+
+if __name__ == "__main__":
+    main()