wehub-resource-sync
a789495a98
FreeBSD Smoke / FreeBSD Smoke (x86_64) (push) Has been cancelled
CI / Quality Guardrails (push) Has been cancelled
CI / Build & Test (macos-latest) (push) Has been cancelled
CI / Build & Test (ubuntu-latest) (push) Has been cancelled
CI / Build & Test (windows-latest) (push) Has been cancelled
CI / Format (push) Has been cancelled
CI / PowerShell Syntax (push) Has been cancelled
CI / Windows Cross-Target Check (Linux) (push) Has been cancelled
476 lines
16 KiB
Python
476 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import pty
|
|
import re
|
|
import select
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
from dataclasses import asdict, dataclass
|
|
from pathlib import Path
|
|
|
|
ANSI_RE = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~]|\][^\x1b\x07]*(?:\x07|\x1b\\))")
|
|
PROBE = "jqx92"
|
|
DEFAULT_TIMEOUT_S = 20.0
|
|
DEFAULT_SETTLE_S = 1.0
|
|
DEFAULT_TOOLS = [
|
|
"jcode_memory_off",
|
|
"jcode_memory_on",
|
|
"pi",
|
|
"codex",
|
|
"opencode",
|
|
"copilot_cli",
|
|
"cursor_agent",
|
|
"claude_code",
|
|
"antigravity_cli",
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class ToolSpec:
|
|
name: str
|
|
argv: list[str]
|
|
version_argv: list[str]
|
|
env: dict[str, str] | None = None
|
|
jcode: bool = False
|
|
|
|
|
|
@dataclass
|
|
class SessionLaunch:
|
|
root_pid: int
|
|
pgid: int
|
|
master_fd: int
|
|
ready: bool
|
|
input_ready: bool
|
|
excerpt: str | None
|
|
seconds_to_visible: float | None
|
|
seconds_to_input_ready: float | None
|
|
buffer_excerpt: str | None
|
|
|
|
|
|
@dataclass
|
|
class ToolRunResult:
|
|
tool: str
|
|
sessions: int
|
|
pss_mb: float
|
|
process_count: int
|
|
version: str
|
|
notes: list[str]
|
|
|
|
|
|
def shutil_which(name: str) -> str | None:
|
|
return subprocess.run(
|
|
["bash", "-lc", f"command -v {name}"], capture_output=True, text=True, check=False
|
|
).stdout.strip() or None
|
|
|
|
|
|
def detect_pi_bin() -> str:
|
|
direct = shutil_which("pi")
|
|
if direct:
|
|
return direct
|
|
prefix = subprocess.check_output(["npm", "prefix", "-g"], text=True).strip()
|
|
candidate = Path(prefix) / "bin" / "pi"
|
|
if candidate.exists():
|
|
return str(candidate)
|
|
raise FileNotFoundError("could not find pi binary")
|
|
|
|
|
|
def build_specs() -> dict[str, ToolSpec]:
|
|
jcode = shutil.which("jcode") or str(Path.home() / ".local/bin/jcode")
|
|
codex = shutil.which("codex") or "/usr/bin/codex"
|
|
opencode = shutil.which("opencode") or "/usr/bin/opencode"
|
|
copilot = shutil.which("copilot") or str(Path.home() / ".local/bin/copilot")
|
|
cursor_agent = shutil.which("cursor-agent") or str(Path.home() / ".local/bin/cursor-agent")
|
|
claude = shutil.which("claude") or str(Path.home() / ".local/bin/claude")
|
|
agy = shutil.which("agy") or str(Path.home() / ".local/bin/agy")
|
|
specs = {
|
|
"jcode_memory_off": ToolSpec(
|
|
name="jcode_memory_off",
|
|
argv=[jcode, "--no-update", "--no-selfdev"],
|
|
version_argv=[jcode, "version"],
|
|
env={"JCODE_NO_TELEMETRY": "1", "JCODE_MEMORY_ENABLED": "0"},
|
|
jcode=True,
|
|
),
|
|
"jcode_memory_on": ToolSpec(
|
|
name="jcode_memory_on",
|
|
argv=[jcode, "--no-update", "--no-selfdev"],
|
|
version_argv=[jcode, "version"],
|
|
env={"JCODE_NO_TELEMETRY": "1", "JCODE_MEMORY_ENABLED": "1"},
|
|
jcode=True,
|
|
),
|
|
"pi": ToolSpec(
|
|
name="pi",
|
|
argv=[detect_pi_bin()],
|
|
version_argv=[detect_pi_bin(), "--version"],
|
|
),
|
|
"codex": ToolSpec(
|
|
name="codex",
|
|
argv=[codex],
|
|
version_argv=[codex, "--version"],
|
|
),
|
|
"opencode": ToolSpec(
|
|
name="opencode",
|
|
argv=[opencode],
|
|
version_argv=[opencode, "--version"],
|
|
),
|
|
"copilot_cli": ToolSpec(
|
|
name="copilot_cli",
|
|
argv=[copilot],
|
|
version_argv=[copilot, "--version"],
|
|
),
|
|
"cursor_agent": ToolSpec(
|
|
name="cursor_agent",
|
|
argv=[cursor_agent],
|
|
version_argv=[cursor_agent, "--version"],
|
|
),
|
|
"claude_code": ToolSpec(
|
|
name="claude_code",
|
|
argv=[claude],
|
|
version_argv=[claude, "--version"],
|
|
),
|
|
"antigravity_cli": ToolSpec(
|
|
name="antigravity_cli",
|
|
argv=[agy],
|
|
version_argv=[agy, "--version"],
|
|
),
|
|
}
|
|
return specs
|
|
|
|
|
|
def reply_queries(master_fd: int, buffer: bytes) -> bytes:
|
|
replies = [
|
|
(b"\x1b[6n", b"\x1b[1;1R"),
|
|
(b"\x1b[c", b"\x1b[?62;c"),
|
|
(b"\x1b]10;?\x1b\\", b"\x1b]10;rgb:ffff/ffff/ffff\x1b\\"),
|
|
(b"\x1b]11;?\x1b\\", b"\x1b]11;rgb:0000/0000/0000\x1b\\"),
|
|
(b"\x1b]10;?\x07", b"\x1b]10;rgb:ffff/ffff/ffff\x07"),
|
|
(b"\x1b]11;?\x07", b"\x1b]11;rgb:0000/0000/0000\x07"),
|
|
(b"\x1b]4;0;?\x07", b"\x1b]4;0;rgb:0000/0000/0000\x07"),
|
|
(b"\x1b[14t", b"\x1b[4;600;800t"),
|
|
(b"\x1b[16t", b"\x1b[6;16;8t"),
|
|
(b"\x1b[18t", b"\x1b[8;24;80t"),
|
|
(b"\x1b[?1016$p", b"\x1b[?1016;1$y"),
|
|
(b"\x1b[?2027$p", b"\x1b[?2027;1$y"),
|
|
(b"\x1b[?2031$p", b"\x1b[?2031;1$y"),
|
|
(b"\x1b[?1004$p", b"\x1b[?1004;1$y"),
|
|
(b"\x1b[?2004$p", b"\x1b[?2004;1$y"),
|
|
(b"\x1b[?2026$p", b"\x1b[?2026;1$y"),
|
|
]
|
|
changed = True
|
|
while changed:
|
|
changed = False
|
|
for query, response in replies:
|
|
if query in buffer:
|
|
os.write(master_fd, response)
|
|
buffer = buffer.replace(query, b"")
|
|
changed = True
|
|
return buffer
|
|
|
|
|
|
def strip_ansi(text: str) -> str:
|
|
return ANSI_RE.sub("", text).replace("\r", "\n")
|
|
|
|
|
|
def first_meaningful_line(text: str) -> str | None:
|
|
for raw_line in text.splitlines():
|
|
line = " ".join(raw_line.split())
|
|
if not line:
|
|
continue
|
|
alnum_count = sum(ch.isalnum() for ch in line)
|
|
if alnum_count >= 3 and len(line) >= 4:
|
|
return line[:160]
|
|
return None
|
|
|
|
|
|
def wait_for_socket(path: str, timeout_s: float) -> bool:
|
|
deadline = time.time() + timeout_s
|
|
while time.time() < deadline:
|
|
if os.path.exists(path):
|
|
try:
|
|
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
sock.connect(path)
|
|
sock.close()
|
|
return True
|
|
except OSError:
|
|
pass
|
|
time.sleep(0.05)
|
|
return False
|
|
|
|
|
|
def launch_interactive(argv: list[str], cwd: Path, env: dict[str, str], timeout_s: float, settle_s: float) -> SessionLaunch:
|
|
master_fd, slave_fd = pty.openpty()
|
|
proc = subprocess.Popen(
|
|
argv,
|
|
cwd=str(cwd),
|
|
env=env,
|
|
stdin=slave_fd,
|
|
stdout=slave_fd,
|
|
stderr=slave_fd,
|
|
preexec_fn=os.setsid,
|
|
)
|
|
os.close(slave_fd)
|
|
os.set_blocking(master_fd, False)
|
|
start = time.perf_counter()
|
|
buf = b""
|
|
ready = False
|
|
input_ready = False
|
|
probe_sent = False
|
|
excerpt = None
|
|
while time.perf_counter() - start < timeout_s:
|
|
rlist, _, _ = select.select([master_fd], [], [], 0.05)
|
|
if rlist:
|
|
try:
|
|
chunk = os.read(master_fd, 65536)
|
|
except BlockingIOError:
|
|
chunk = b""
|
|
if chunk:
|
|
buf += chunk
|
|
buf = reply_queries(master_fd, buf)
|
|
plain = strip_ansi(buf.decode("utf-8", "replace"))
|
|
excerpt = first_meaningful_line(plain)
|
|
if excerpt:
|
|
ready = True
|
|
if not probe_sent:
|
|
try:
|
|
os.write(master_fd, PROBE.encode())
|
|
probe_sent = True
|
|
except OSError:
|
|
break
|
|
if probe_sent and PROBE in plain:
|
|
input_ready = True
|
|
break
|
|
if proc.poll() is not None:
|
|
break
|
|
if input_ready or ready:
|
|
time.sleep(settle_s)
|
|
elapsed = time.perf_counter() - start
|
|
return SessionLaunch(
|
|
root_pid=proc.pid,
|
|
pgid=os.getpgid(proc.pid),
|
|
master_fd=master_fd,
|
|
ready=ready,
|
|
input_ready=input_ready,
|
|
excerpt=excerpt,
|
|
seconds_to_visible=elapsed if ready else None,
|
|
seconds_to_input_ready=elapsed if input_ready else None,
|
|
buffer_excerpt=(strip_ansi(buf.decode("utf-8", "replace"))[:300] or None),
|
|
)
|
|
|
|
|
|
def iter_proc_stat() -> dict[int, tuple[int, int]]:
|
|
out: dict[int, tuple[int, int]] = {}
|
|
for entry in Path("/proc").iterdir():
|
|
if not entry.name.isdigit():
|
|
continue
|
|
try:
|
|
stat = (entry / "stat").read_text()
|
|
except Exception:
|
|
continue
|
|
try:
|
|
close = stat.rfind(")")
|
|
rest = stat[close + 2 :].split()
|
|
ppid = int(rest[1])
|
|
pgid = int(rest[2])
|
|
out[int(entry.name)] = (ppid, pgid)
|
|
except Exception:
|
|
continue
|
|
return out
|
|
|
|
|
|
def collect_descendants(root_pids: list[int]) -> set[int]:
|
|
ppid_of = iter_proc_stat()
|
|
children: dict[int, list[int]] = {}
|
|
for pid, (ppid, _pgid) in ppid_of.items():
|
|
children.setdefault(ppid, []).append(pid)
|
|
seen: set[int] = set()
|
|
stack = list(root_pids)
|
|
while stack:
|
|
pid = stack.pop()
|
|
if pid in seen:
|
|
continue
|
|
seen.add(pid)
|
|
stack.extend(children.get(pid, []))
|
|
return seen
|
|
|
|
|
|
def collect_process_group_pids(pgids: list[int]) -> set[int]:
|
|
proc_map = iter_proc_stat()
|
|
wanted = set(pgids)
|
|
return {pid for pid, (_ppid, pgid) in proc_map.items() if pgid in wanted}
|
|
|
|
|
|
def read_pss_mb(pid: int) -> float | None:
|
|
path = Path(f"/proc/{pid}/smaps_rollup")
|
|
try:
|
|
for line in path.read_text().splitlines():
|
|
if line.startswith("Pss:"):
|
|
return int(line.split()[1]) / 1024.0
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
|
|
def sum_tree_pss(root_pids: list[int], pgids: list[int]) -> tuple[float, int]:
|
|
all_pids = collect_descendants(root_pids) | collect_process_group_pids(pgids)
|
|
total = 0.0
|
|
counted = 0
|
|
for pid in sorted(all_pids):
|
|
pss = read_pss_mb(pid)
|
|
if pss is None:
|
|
continue
|
|
total += pss
|
|
counted += 1
|
|
return round(total, 1), counted
|
|
|
|
|
|
def terminate_pgroup(pgid: int) -> None:
|
|
for sig in (signal.SIGTERM, signal.SIGKILL):
|
|
try:
|
|
os.killpg(pgid, sig)
|
|
time.sleep(0.2)
|
|
except ProcessLookupError:
|
|
return
|
|
|
|
|
|
def version_for(spec: ToolSpec) -> str:
|
|
proc = subprocess.run(spec.version_argv, capture_output=True, text=True, check=False)
|
|
output = (proc.stdout + proc.stderr).strip().splitlines()
|
|
return output[0] if output else f"exit {proc.returncode}"
|
|
|
|
|
|
def run_tool(spec: ToolSpec, sessions: int, cwd: Path, timeout_s: float, settle_s: float) -> ToolRunResult:
|
|
notes: list[str] = []
|
|
version = version_for(spec)
|
|
launches: list[SessionLaunch] = []
|
|
cleanup_pgids: list[int] = []
|
|
temp_root: str | None = None
|
|
try:
|
|
if spec.jcode:
|
|
temp_root = tempfile.mkdtemp(prefix="jcode-memory-bench-")
|
|
env = os.environ.copy()
|
|
if spec.env:
|
|
env.update(spec.env)
|
|
env["JCODE_HOME"] = os.path.join(temp_root, "home")
|
|
env["JCODE_RUNTIME_DIR"] = os.path.join(temp_root, "run")
|
|
env["JCODE_TEMP_SERVER"] = "1"
|
|
env["JCODE_SERVER_OWNER_PID"] = str(os.getpid())
|
|
os.makedirs(env["JCODE_HOME"], exist_ok=True)
|
|
os.makedirs(env["JCODE_RUNTIME_DIR"], exist_ok=True)
|
|
for auth_name in (
|
|
"anthropic-auth.json",
|
|
"openai-auth.json",
|
|
"antigravity_oauth.json",
|
|
"gemini_oauth.json",
|
|
"config.toml",
|
|
):
|
|
real_auth = Path.home() / ".jcode" / auth_name
|
|
bench_auth = Path(env["JCODE_HOME"]) / auth_name
|
|
if real_auth.exists() and not bench_auth.exists():
|
|
bench_auth.symlink_to(real_auth)
|
|
if spec.name == "jcode_memory_on":
|
|
real_models = Path.home() / ".jcode" / "models"
|
|
bench_models = Path(env["JCODE_HOME"]) / "models"
|
|
if real_models.exists() and not bench_models.exists():
|
|
bench_models.symlink_to(real_models)
|
|
socket_path = os.path.join(env["JCODE_RUNTIME_DIR"], "bench.sock")
|
|
server_proc = subprocess.Popen(
|
|
[spec.argv[0], "--no-update", "--no-selfdev", "serve", "--socket", socket_path],
|
|
cwd=str(cwd),
|
|
env=env,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
preexec_fn=os.setsid,
|
|
)
|
|
cleanup_pgids.append(os.getpgid(server_proc.pid))
|
|
if not wait_for_socket(socket_path, timeout_s):
|
|
raise RuntimeError("jcode server did not become ready")
|
|
if spec.name == "jcode_memory_on":
|
|
time.sleep(max(settle_s, 5.0))
|
|
per_session_settle = max(settle_s, 2.0) if spec.name == "jcode_memory_on" else settle_s
|
|
for _ in range(sessions):
|
|
launches.append(
|
|
launch_interactive(
|
|
[spec.argv[0], "--no-update", "--no-selfdev", "--socket", socket_path],
|
|
cwd,
|
|
env,
|
|
timeout_s,
|
|
per_session_settle,
|
|
)
|
|
)
|
|
cleanup_pgids.append(launches[-1].pgid)
|
|
root_pids = [server_proc.pid] + [launch.root_pid for launch in launches]
|
|
sample_pgids = cleanup_pgids.copy()
|
|
else:
|
|
env = os.environ.copy()
|
|
if spec.env:
|
|
env.update(spec.env)
|
|
for _ in range(sessions):
|
|
launches.append(launch_interactive(spec.argv, cwd, env, timeout_s, settle_s))
|
|
cleanup_pgids.append(launches[-1].pgid)
|
|
root_pids = [launch.root_pid for launch in launches]
|
|
sample_pgids = cleanup_pgids.copy()
|
|
|
|
for idx, launch in enumerate(launches, start=1):
|
|
if not launch.ready:
|
|
notes.append(f"session {idx}: no meaningful screen content before timeout")
|
|
elif launch.excerpt:
|
|
notes.append(f"session {idx}: {launch.excerpt}")
|
|
pss_mb, process_count = sum_tree_pss(root_pids, sample_pgids)
|
|
return ToolRunResult(
|
|
tool=spec.name,
|
|
sessions=sessions,
|
|
pss_mb=pss_mb,
|
|
process_count=process_count,
|
|
version=version,
|
|
notes=notes,
|
|
)
|
|
finally:
|
|
for launch in launches:
|
|
try:
|
|
os.close(launch.master_fd)
|
|
except Exception:
|
|
pass
|
|
for pgid in reversed(cleanup_pgids):
|
|
terminate_pgroup(pgid)
|
|
if temp_root:
|
|
shutil.rmtree(temp_root, ignore_errors=True)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Benchmark interactive CLI memory using process-tree PSS")
|
|
parser.add_argument("--sessions", type=int, required=True)
|
|
parser.add_argument("--tools", nargs="*", default=DEFAULT_TOOLS)
|
|
parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT_S)
|
|
parser.add_argument("--settle", type=float, default=DEFAULT_SETTLE_S)
|
|
parser.add_argument("--cwd", default=os.getcwd())
|
|
parser.add_argument("--json-out", default=None)
|
|
args = parser.parse_args()
|
|
|
|
specs = build_specs()
|
|
cwd = Path(args.cwd).resolve()
|
|
results = []
|
|
for name in args.tools:
|
|
spec = specs[name]
|
|
print(f"=== {name} ({args.sessions} session{'s' if args.sessions != 1 else ''}) ===", flush=True)
|
|
result = run_tool(spec, args.sessions, cwd, args.timeout, args.settle)
|
|
print(json.dumps(asdict(result), indent=2), flush=True)
|
|
results.append(asdict(result))
|
|
payload = {"cwd": str(cwd), "sessions": args.sessions, "results": results}
|
|
if args.json_out:
|
|
Path(args.json_out).write_text(json.dumps(payload, indent=2))
|
|
else:
|
|
print(json.dumps(payload, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|