chore: import upstream snapshot with attribution

scripts/bench_memory_cli.py

@@ -0,0 +1,475 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import pty
+import re
+import select
+import shutil
+import signal
+import socket
+import subprocess
+import tempfile
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+ANSI_RE = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~]|\][^\x1b\x07]*(?:\x07|\x1b\\))")
+PROBE = "jqx92"
+DEFAULT_TIMEOUT_S = 20.0
+DEFAULT_SETTLE_S = 1.0
+DEFAULT_TOOLS = [
+    "jcode_memory_off",
+    "jcode_memory_on",
+    "pi",
+    "codex",
+    "opencode",
+    "copilot_cli",
+    "cursor_agent",
+    "claude_code",
+    "antigravity_cli",
+]
+
+
+@dataclass
+class ToolSpec:
+    name: str
+    argv: list[str]
+    version_argv: list[str]
+    env: dict[str, str] | None = None
+    jcode: bool = False
+
+
+@dataclass
+class SessionLaunch:
+    root_pid: int
+    pgid: int
+    master_fd: int
+    ready: bool
+    input_ready: bool
+    excerpt: str | None
+    seconds_to_visible: float | None
+    seconds_to_input_ready: float | None
+    buffer_excerpt: str | None
+
+
+@dataclass
+class ToolRunResult:
+    tool: str
+    sessions: int
+    pss_mb: float
+    process_count: int
+    version: str
+    notes: list[str]
+
+
+def shutil_which(name: str) -> str | None:
+    return subprocess.run(
+        ["bash", "-lc", f"command -v {name}"], capture_output=True, text=True, check=False
+    ).stdout.strip() or None
+
+
+def detect_pi_bin() -> str:
+    direct = shutil_which("pi")
+    if direct:
+        return direct
+    prefix = subprocess.check_output(["npm", "prefix", "-g"], text=True).strip()
+    candidate = Path(prefix) / "bin" / "pi"
+    if candidate.exists():
+        return str(candidate)
+    raise FileNotFoundError("could not find pi binary")
+
+
+def build_specs() -> dict[str, ToolSpec]:
+    jcode = shutil.which("jcode") or str(Path.home() / ".local/bin/jcode")
+    codex = shutil.which("codex") or "/usr/bin/codex"
+    opencode = shutil.which("opencode") or "/usr/bin/opencode"
+    copilot = shutil.which("copilot") or str(Path.home() / ".local/bin/copilot")
+    cursor_agent = shutil.which("cursor-agent") or str(Path.home() / ".local/bin/cursor-agent")
+    claude = shutil.which("claude") or str(Path.home() / ".local/bin/claude")
+    agy = shutil.which("agy") or str(Path.home() / ".local/bin/agy")
+    specs = {
+        "jcode_memory_off": ToolSpec(
+            name="jcode_memory_off",
+            argv=[jcode, "--no-update", "--no-selfdev"],
+            version_argv=[jcode, "version"],
+            env={"JCODE_NO_TELEMETRY": "1", "JCODE_MEMORY_ENABLED": "0"},
+            jcode=True,
+        ),
+        "jcode_memory_on": ToolSpec(
+            name="jcode_memory_on",
+            argv=[jcode, "--no-update", "--no-selfdev"],
+            version_argv=[jcode, "version"],
+            env={"JCODE_NO_TELEMETRY": "1", "JCODE_MEMORY_ENABLED": "1"},
+            jcode=True,
+        ),
+        "pi": ToolSpec(
+            name="pi",
+            argv=[detect_pi_bin()],
+            version_argv=[detect_pi_bin(), "--version"],
+        ),
+        "codex": ToolSpec(
+            name="codex",
+            argv=[codex],
+            version_argv=[codex, "--version"],
+        ),
+        "opencode": ToolSpec(
+            name="opencode",
+            argv=[opencode],
+            version_argv=[opencode, "--version"],
+        ),
+        "copilot_cli": ToolSpec(
+            name="copilot_cli",
+            argv=[copilot],
+            version_argv=[copilot, "--version"],
+        ),
+        "cursor_agent": ToolSpec(
+            name="cursor_agent",
+            argv=[cursor_agent],
+            version_argv=[cursor_agent, "--version"],
+        ),
+        "claude_code": ToolSpec(
+            name="claude_code",
+            argv=[claude],
+            version_argv=[claude, "--version"],
+        ),
+        "antigravity_cli": ToolSpec(
+            name="antigravity_cli",
+            argv=[agy],
+            version_argv=[agy, "--version"],
+        ),
+    }
+    return specs
+
+
+def reply_queries(master_fd: int, buffer: bytes) -> bytes:
+    replies = [
+        (b"\x1b[6n", b"\x1b[1;1R"),
+        (b"\x1b[c", b"\x1b[?62;c"),
+        (b"\x1b]10;?\x1b\\", b"\x1b]10;rgb:ffff/ffff/ffff\x1b\\"),
+        (b"\x1b]11;?\x1b\\", b"\x1b]11;rgb:0000/0000/0000\x1b\\"),
+        (b"\x1b]10;?\x07", b"\x1b]10;rgb:ffff/ffff/ffff\x07"),
+        (b"\x1b]11;?\x07", b"\x1b]11;rgb:0000/0000/0000\x07"),
+        (b"\x1b]4;0;?\x07", b"\x1b]4;0;rgb:0000/0000/0000\x07"),
+        (b"\x1b[14t", b"\x1b[4;600;800t"),
+        (b"\x1b[16t", b"\x1b[6;16;8t"),
+        (b"\x1b[18t", b"\x1b[8;24;80t"),
+        (b"\x1b[?1016$p", b"\x1b[?1016;1$y"),
+        (b"\x1b[?2027$p", b"\x1b[?2027;1$y"),
+        (b"\x1b[?2031$p", b"\x1b[?2031;1$y"),
+        (b"\x1b[?1004$p", b"\x1b[?1004;1$y"),
+        (b"\x1b[?2004$p", b"\x1b[?2004;1$y"),
+        (b"\x1b[?2026$p", b"\x1b[?2026;1$y"),
+    ]
+    changed = True
+    while changed:
+        changed = False
+        for query, response in replies:
+            if query in buffer:
+                os.write(master_fd, response)
+                buffer = buffer.replace(query, b"")
+                changed = True
+    return buffer
+
+
+def strip_ansi(text: str) -> str:
+    return ANSI_RE.sub("", text).replace("\r", "\n")
+
+
+def first_meaningful_line(text: str) -> str | None:
+    for raw_line in text.splitlines():
+        line = " ".join(raw_line.split())
+        if not line:
+            continue
+        alnum_count = sum(ch.isalnum() for ch in line)
+        if alnum_count >= 3 and len(line) >= 4:
+            return line[:160]
+    return None
+
+
+def wait_for_socket(path: str, timeout_s: float) -> bool:
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        if os.path.exists(path):
+            try:
+                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                sock.connect(path)
+                sock.close()
+                return True
+            except OSError:
+                pass
+        time.sleep(0.05)
+    return False
+
+
+def launch_interactive(argv: list[str], cwd: Path, env: dict[str, str], timeout_s: float, settle_s: float) -> SessionLaunch:
+    master_fd, slave_fd = pty.openpty()
+    proc = subprocess.Popen(
+        argv,
+        cwd=str(cwd),
+        env=env,
+        stdin=slave_fd,
+        stdout=slave_fd,
+        stderr=slave_fd,
+        preexec_fn=os.setsid,
+    )
+    os.close(slave_fd)
+    os.set_blocking(master_fd, False)
+    start = time.perf_counter()
+    buf = b""
+    ready = False
+    input_ready = False
+    probe_sent = False
+    excerpt = None
+    while time.perf_counter() - start < timeout_s:
+        rlist, _, _ = select.select([master_fd], [], [], 0.05)
+        if rlist:
+            try:
+                chunk = os.read(master_fd, 65536)
+            except BlockingIOError:
+                chunk = b""
+            if chunk:
+                buf += chunk
+                buf = reply_queries(master_fd, buf)
+                plain = strip_ansi(buf.decode("utf-8", "replace"))
+                excerpt = first_meaningful_line(plain)
+                if excerpt:
+                    ready = True
+                    if not probe_sent:
+                        try:
+                            os.write(master_fd, PROBE.encode())
+                            probe_sent = True
+                        except OSError:
+                            break
+                if probe_sent and PROBE in plain:
+                    input_ready = True
+                    break
+        if proc.poll() is not None:
+            break
+    if input_ready or ready:
+        time.sleep(settle_s)
+    elapsed = time.perf_counter() - start
+    return SessionLaunch(
+        root_pid=proc.pid,
+        pgid=os.getpgid(proc.pid),
+        master_fd=master_fd,
+        ready=ready,
+        input_ready=input_ready,
+        excerpt=excerpt,
+        seconds_to_visible=elapsed if ready else None,
+        seconds_to_input_ready=elapsed if input_ready else None,
+        buffer_excerpt=(strip_ansi(buf.decode("utf-8", "replace"))[:300] or None),
+    )
+
+
+def iter_proc_stat() -> dict[int, tuple[int, int]]:
+    out: dict[int, tuple[int, int]] = {}
+    for entry in Path("/proc").iterdir():
+        if not entry.name.isdigit():
+            continue
+        try:
+            stat = (entry / "stat").read_text()
+        except Exception:
+            continue
+        try:
+            close = stat.rfind(")")
+            rest = stat[close + 2 :].split()
+            ppid = int(rest[1])
+            pgid = int(rest[2])
+            out[int(entry.name)] = (ppid, pgid)
+        except Exception:
+            continue
+    return out
+
+
+def collect_descendants(root_pids: list[int]) -> set[int]:
+    ppid_of = iter_proc_stat()
+    children: dict[int, list[int]] = {}
+    for pid, (ppid, _pgid) in ppid_of.items():
+        children.setdefault(ppid, []).append(pid)
+    seen: set[int] = set()
+    stack = list(root_pids)
+    while stack:
+        pid = stack.pop()
+        if pid in seen:
+            continue
+        seen.add(pid)
+        stack.extend(children.get(pid, []))
+    return seen
+
+
+def collect_process_group_pids(pgids: list[int]) -> set[int]:
+    proc_map = iter_proc_stat()
+    wanted = set(pgids)
+    return {pid for pid, (_ppid, pgid) in proc_map.items() if pgid in wanted}
+
+
+def read_pss_mb(pid: int) -> float | None:
+    path = Path(f"/proc/{pid}/smaps_rollup")
+    try:
+        for line in path.read_text().splitlines():
+            if line.startswith("Pss:"):
+                return int(line.split()[1]) / 1024.0
+    except Exception:
+        return None
+    return None
+
+
+def sum_tree_pss(root_pids: list[int], pgids: list[int]) -> tuple[float, int]:
+    all_pids = collect_descendants(root_pids) | collect_process_group_pids(pgids)
+    total = 0.0
+    counted = 0
+    for pid in sorted(all_pids):
+        pss = read_pss_mb(pid)
+        if pss is None:
+            continue
+        total += pss
+        counted += 1
+    return round(total, 1), counted
+
+
+def terminate_pgroup(pgid: int) -> None:
+    for sig in (signal.SIGTERM, signal.SIGKILL):
+        try:
+            os.killpg(pgid, sig)
+            time.sleep(0.2)
+        except ProcessLookupError:
+            return
+
+
+def version_for(spec: ToolSpec) -> str:
+    proc = subprocess.run(spec.version_argv, capture_output=True, text=True, check=False)
+    output = (proc.stdout + proc.stderr).strip().splitlines()
+    return output[0] if output else f"exit {proc.returncode}"
+
+
+def run_tool(spec: ToolSpec, sessions: int, cwd: Path, timeout_s: float, settle_s: float) -> ToolRunResult:
+    notes: list[str] = []
+    version = version_for(spec)
+    launches: list[SessionLaunch] = []
+    cleanup_pgids: list[int] = []
+    temp_root: str | None = None
+    try:
+        if spec.jcode:
+            temp_root = tempfile.mkdtemp(prefix="jcode-memory-bench-")
+            env = os.environ.copy()
+            if spec.env:
+                env.update(spec.env)
+            env["JCODE_HOME"] = os.path.join(temp_root, "home")
+            env["JCODE_RUNTIME_DIR"] = os.path.join(temp_root, "run")
+            env["JCODE_TEMP_SERVER"] = "1"
+            env["JCODE_SERVER_OWNER_PID"] = str(os.getpid())
+            os.makedirs(env["JCODE_HOME"], exist_ok=True)
+            os.makedirs(env["JCODE_RUNTIME_DIR"], exist_ok=True)
+            for auth_name in (
+                "anthropic-auth.json",
+                "openai-auth.json",
+                "antigravity_oauth.json",
+                "gemini_oauth.json",
+                "config.toml",
+            ):
+                real_auth = Path.home() / ".jcode" / auth_name
+                bench_auth = Path(env["JCODE_HOME"]) / auth_name
+                if real_auth.exists() and not bench_auth.exists():
+                    bench_auth.symlink_to(real_auth)
+            if spec.name == "jcode_memory_on":
+                real_models = Path.home() / ".jcode" / "models"
+                bench_models = Path(env["JCODE_HOME"]) / "models"
+                if real_models.exists() and not bench_models.exists():
+                    bench_models.symlink_to(real_models)
+            socket_path = os.path.join(env["JCODE_RUNTIME_DIR"], "bench.sock")
+            server_proc = subprocess.Popen(
+                [spec.argv[0], "--no-update", "--no-selfdev", "serve", "--socket", socket_path],
+                cwd=str(cwd),
+                env=env,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                preexec_fn=os.setsid,
+            )
+            cleanup_pgids.append(os.getpgid(server_proc.pid))
+            if not wait_for_socket(socket_path, timeout_s):
+                raise RuntimeError("jcode server did not become ready")
+            if spec.name == "jcode_memory_on":
+                time.sleep(max(settle_s, 5.0))
+            per_session_settle = max(settle_s, 2.0) if spec.name == "jcode_memory_on" else settle_s
+            for _ in range(sessions):
+                launches.append(
+                    launch_interactive(
+                        [spec.argv[0], "--no-update", "--no-selfdev", "--socket", socket_path],
+                        cwd,
+                        env,
+                        timeout_s,
+                        per_session_settle,
+                    )
+                )
+                cleanup_pgids.append(launches[-1].pgid)
+            root_pids = [server_proc.pid] + [launch.root_pid for launch in launches]
+            sample_pgids = cleanup_pgids.copy()
+        else:
+            env = os.environ.copy()
+            if spec.env:
+                env.update(spec.env)
+            for _ in range(sessions):
+                launches.append(launch_interactive(spec.argv, cwd, env, timeout_s, settle_s))
+                cleanup_pgids.append(launches[-1].pgid)
+            root_pids = [launch.root_pid for launch in launches]
+            sample_pgids = cleanup_pgids.copy()
+
+        for idx, launch in enumerate(launches, start=1):
+            if not launch.ready:
+                notes.append(f"session {idx}: no meaningful screen content before timeout")
+            elif launch.excerpt:
+                notes.append(f"session {idx}: {launch.excerpt}")
+        pss_mb, process_count = sum_tree_pss(root_pids, sample_pgids)
+        return ToolRunResult(
+            tool=spec.name,
+            sessions=sessions,
+            pss_mb=pss_mb,
+            process_count=process_count,
+            version=version,
+            notes=notes,
+        )
+    finally:
+        for launch in launches:
+            try:
+                os.close(launch.master_fd)
+            except Exception:
+                pass
+        for pgid in reversed(cleanup_pgids):
+            terminate_pgroup(pgid)
+        if temp_root:
+            shutil.rmtree(temp_root, ignore_errors=True)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Benchmark interactive CLI memory using process-tree PSS")
+    parser.add_argument("--sessions", type=int, required=True)
+    parser.add_argument("--tools", nargs="*", default=DEFAULT_TOOLS)
+    parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT_S)
+    parser.add_argument("--settle", type=float, default=DEFAULT_SETTLE_S)
+    parser.add_argument("--cwd", default=os.getcwd())
+    parser.add_argument("--json-out", default=None)
+    args = parser.parse_args()
+
+    specs = build_specs()
+    cwd = Path(args.cwd).resolve()
+    results = []
+    for name in args.tools:
+        spec = specs[name]
+        print(f"=== {name} ({args.sessions} session{'s' if args.sessions != 1 else ''}) ===", flush=True)
+        result = run_tool(spec, args.sessions, cwd, args.timeout, args.settle)
+        print(json.dumps(asdict(result), indent=2), flush=True)
+        results.append(asdict(result))
+    payload = {"cwd": str(cwd), "sessions": args.sessions, "results": results}
+    if args.json_out:
+        Path(args.json_out).write_text(json.dumps(payload, indent=2))
+    else:
+        print(json.dumps(payload, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())