1jehuang--jcode/scripts/test_caching_detailed.py

#!/usr/bin/env python3
"""
Test caching behavior across multiple turns with tool usage.
"""

import socket
import json
import time
import os

DEBUG_SOCKET = f"/run/user/{os.getuid()}/jcode-debug.sock"

def send_cmd(sock, cmd, session_id=None, timeout=120):
    """Send a debug command and get response."""
    req = {"type": "debug_command", "id": 1, "command": cmd}
    if session_id:
        req["session_id"] = session_id
    sock.send((json.dumps(req) + '\n').encode())
    sock.settimeout(timeout)

    data = b""
    while True:
        chunk = sock.recv(65536)
        if not chunk:
            break
        data += chunk
        if b'\n' in data:
            break

    resp = json.loads(data.decode().strip())
    return resp.get('ok', False), resp.get('output', ''), resp.get('error', '')


def main():
    print("=" * 70)
    print("Multi-turn Caching Test")
    print("=" * 70)

    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.connect(DEBUG_SOCKET)

    # Create a test session
    ok, output, err = send_cmd(sock, "create_session:/tmp/cache-test")
    if not ok:
        print(f"Failed to create session: {err}")
        return

    session_data = json.loads(output)
    session_id = session_data.get("session_id")
    print(f"Session: {session_id}\n")

    # Define a multi-step task
    messages = [
        "List the files in /tmp and tell me how many there are.",
        "Now read the contents of /etc/hostname",
        "What's the current date and time? Use the bash tool to run 'date'.",
        "Summarize what you found in the previous steps."
    ]

    total_input = 0
    total_output = 0
    total_cache_read = 0
    total_cache_creation = 0

    for i, msg in enumerate(messages):
        print(f"\n{'='*70}")
        print(f"Turn {i+1}: {msg[:50]}...")
        print("=" * 70)

        start = time.time()
        ok, response, err = send_cmd(sock, f"message:{msg}", session_id, timeout=180)
        elapsed = time.time() - start

        if not ok:
            print(f"Error: {err}")
            continue

        print(f"Time: {elapsed:.2f}s")
        print(f"Response: {response[:200]}...")

        # Get usage
        ok, usage_output, _ = send_cmd(sock, "usage", session_id)
        if ok:
            try:
                usage = json.loads(usage_output)
                input_tokens = usage.get('input_tokens', 0)
                output_tokens = usage.get('output_tokens', 0)
                cache_read = usage.get('cache_read_input_tokens') or 0
                cache_creation = usage.get('cache_creation_input_tokens') or 0

                total_input += input_tokens
                total_output += output_tokens
                total_cache_read += cache_read
                total_cache_creation += cache_creation

                print(f"\nUsage this turn:")
                print(f"  Input (non-cached): {input_tokens:,}")
                print(f"  Output: {output_tokens:,}")
                print(f"  Cache read: {cache_read:,}")
                print(f"  Cache creation: {cache_creation:,}")

                # Calculate cache efficiency
                total_input_this_turn = input_tokens + cache_read + cache_creation
                if total_input_this_turn > 0:
                    cache_hit_rate = (cache_read / total_input_this_turn) * 100
                    print(f"  Cache hit rate: {cache_hit_rate:.1f}%")

            except json.JSONDecodeError:
                print(f"Usage parse error: {usage_output}")

    # Summary
    print(f"\n{'='*70}")
    print("SUMMARY")
    print("=" * 70)
    print(f"\nTotal across {len(messages)} turns:")
    print(f"  Input (non-cached): {total_input:,}")
    print(f"  Output: {total_output:,}")
    print(f"  Cache read: {total_cache_read:,}")
    print(f"  Cache creation: {total_cache_creation:,}")

    total_all = total_input + total_cache_read + total_cache_creation
    if total_all > 0:
        overall_cache_rate = (total_cache_read / total_all) * 100
        print(f"\nOverall cache hit rate: {overall_cache_rate:.1f}%")

    # Effective tokens (cache reads at 10%)
    effective = total_input + (total_cache_read * 0.1) + total_output
    print(f"Effective tokens (cache @ 10%): {effective:,.0f}")

    # Check for anomalies
    print(f"\n{'='*70}")
    print("ANALYSIS")
    print("=" * 70)

    if total_cache_creation > total_cache_read:
        print("WARNING: More cache creation than reads - caching may not be working well")
        print(f"  Creation: {total_cache_creation:,} vs Read: {total_cache_read:,}")

    avg_non_cached = total_input / len(messages)
    print(f"\nAverage non-cached input per turn: {avg_non_cached:,.0f}")

    if avg_non_cached > 1000:
        print("NOTE: High non-cached input suggests dynamic content or cache misses")

    # Cleanup
    send_cmd(sock, f"destroy_session:{session_id}")
    sock.close()

    print("\nDone.")


if __name__ == "__main__":
    main()