chore: import upstream snapshot with attribution

2026-07-13 13:01:18 +08:00
commit 979fb22d7c
613 changed files with 113494 additions and 0 deletions
@@ -0,0 +1,111 @@
+# agentmemory-evals
+
+Public benchmarks for agentmemory's hybrid memory stack (BM25 + embeddings + consolidation + graph).
+
+Two families, both reproducible:
+
+- **LongMemEval** — public 500-question retrieval benchmark over multi-session chat
+- **coding-agent-life-v1** — in-house corpus of 15 fictional Claude Code sessions for a Rust CLI project (`shipctl`), with 15 hand-graded queries covering bug fixes, refactors, preferences, and multi-session causal reasoning
+
+## Adapters
+
+| Adapter | Backend | API key needed |
+|---|---|---|
+| `grep` | Tokenized substring match | none |
+| `vector` | OpenAI `text-embedding-3-small` + cosine | `OPENAI_API_KEY` |
+| `agentmemory` | Running agentmemory server, smart-search endpoint | none (auth optional via `AGENTMEMORY_SECRET`) |
+
+## Sandbox first
+
+Running the `agentmemory` adapter against your real `~/.agentmemory` directory pollutes the eval with pre-existing memories AND pollutes your real store with eval test data. Always sandbox.
+
+`eval/scripts/sandbox.sh` spins up a clean agentmemory + iii-engine on ports 3411/3412 with state in `/tmp/agentmemory-eval-sandbox/`, exports `AGENTMEMORY_BASE_URL`, and tears down on exit.
+
+```sh
+source eval/scripts/sandbox.sh
+npm run eval:coding-life -- --adapters grep,agentmemory
+```
+
+Requires iii v0.11.2 on PATH (agentmemory pin). If you already have a different version installed, install the pinned build into `~/.local/bin` and make sure that directory comes first on `PATH`:
+
+```sh
+mkdir -p ~/.local/bin
+curl -fsSL https://github.com/iii-hq/iii/releases/download/iii/v0.11.2/iii-aarch64-apple-darwin.tar.gz | tar -xz -C ~/.local/bin
+export PATH="$HOME/.local/bin:$PATH"  # add to ~/.zshrc or ~/.bashrc for persistence
+```
+
+## Quickstart
+
+### coding-agent-life-v1 (in-house, no download)
+
+```sh
+# grep baseline, no sandbox needed
+npm run eval:coding-life -- --adapters grep
+
+# add agentmemory + vector (sandbox + OpenAI key)
+source eval/scripts/sandbox.sh
+OPENAI_API_KEY=sk-... npm run eval:coding-life -- --adapters grep,vector,agentmemory
+```
+
+### LongMemEval `_s` (public, 278MB download)
+
+```sh
+mkdir -p ~/datasets/longmemeval
+curl -Lo ~/datasets/longmemeval/longmemeval_s.json \
+  https://huggingface.co/datasets/xiaowu0162/longmemeval/resolve/main/longmemeval_s
+
+source eval/scripts/sandbox.sh
+
+# Stratified sample of 10 per type (fast iteration, ~$0.20 OpenAI cost)
+OPENAI_API_KEY=sk-... LONGMEMEVAL_PATH=~/datasets/longmemeval/longmemeval_s.json \
+  npm run eval:longmemeval -- --stratify 10
+
+# Full 500 questions × 3 adapters (~$2 OpenAI cost)
+OPENAI_API_KEY=sk-... LONGMEMEVAL_PATH=~/datasets/longmemeval/longmemeval_s.json \
+  npm run eval:longmemeval
+```
+
+## Repo layout
+
+```text
+eval/
+├── README.md
+├── runner/
+│   ├── types.ts                   Adapter, Question, RankedDoc, ScoreRow
+│   ├── score.ts                   P@K, R@K, aggregation
+│   ├── load.ts                    LongMemEval JSON → Question[]
+│   ├── adapters/
+│   │   ├── grep.ts                tokenized substring baseline
+│   │   ├── vector.ts              OpenAI embeddings + cosine
+│   │   └── agentmemory.ts         POST /agentmemory/{remember,smart-search}
+│   ├── longmemeval.ts             public benchmark runner
+│   └── coding-life.ts             in-house benchmark runner
+└── data/
+    └── coding-agent-life-v1/
+        ├── sessions.json          15 fictional sessions (~6KB)
+        └── queries.json           15 queries with gold session IDs
+```
+
+Reports land in `eval/reports/<bench>/` (gitignored): `scores.ndjson` + `summary.json`.
+
+Published scorecards land in `docs/benchmarks/YYYY-MM-DD-<bench>.md`.
+
+## Writing a new adapter
+
+1. Implement `Adapter<State>` from `eval/runner/types.ts`:
+   ```ts
+   import type { Adapter } from "../types.js";
+   export const myAdapter: Adapter<MyState> = {
+     name: "my-adapter",
+     async init(sessions, config) { /* index */ return state; },
+     async query(q, state, k) { /* search */ return ranked; },
+   };
+   ```
+2. Register in `eval/runner/{longmemeval,coding-life}.ts` `ADAPTERS` map.
+3. Run against `coding-agent-life-v1` to sanity-check before committing OpenAI spend on LongMemEval.
+
+## Why a benchmark for agentmemory
+
+agentmemory ships BM25 + embeddings + consolidation + graph retrieval. Numbers from those layers should be measured against grep/vector baselines so the value of each layer is provable.
+
+The in-house corpus is small on purpose (15 sessions) — covers single-session, multi-session, preference, and temporal question types without taking 15 minutes to run. LongMemEval gives the public-comparison axis.
@@ -0,0 +1,107 @@
+[
+  {
+    "id": "q-001",
+    "type": "single-session-bug",
+    "question": "Where did we land the auth env var precedence fix?",
+    "answer": "PR #11 with SHIPCTL_TOKEN > SHIP_TOKEN > SC_TOKEN precedence",
+    "goldSessionIds": ["sess-001"]
+  },
+  {
+    "id": "q-002",
+    "type": "single-session-infra",
+    "question": "What was the multi-arch Docker fix?",
+    "answer": "Added --platform=$BUILDPLATFORM and BUILDX_PLATFORMS for amd64+arm64",
+    "goldSessionIds": ["sess-002"]
+  },
+  {
+    "id": "q-003",
+    "type": "single-session-refactor",
+    "question": "Where did we consolidate the retry logic?",
+    "answer": "src/retry.rs with exponential backoff base=200ms cap=30s full jitter",
+    "goldSessionIds": ["sess-003"]
+  },
+  {
+    "id": "q-004",
+    "type": "single-session-feature",
+    "question": "Which PR introduced helm chart support?",
+    "answer": "PR #14",
+    "goldSessionIds": ["sess-004"]
+  },
+  {
+    "id": "q-005",
+    "type": "single-session-test",
+    "question": "Which test was flaky on macos and how was it fixed?",
+    "answer": "fs-watcher emits_changekind_file_delete; bumped wait to 1500ms + retry: 2",
+    "goldSessionIds": ["sess-005"]
+  },
+  {
+    "id": "q-006",
+    "type": "single-session-perf",
+    "question": "How did we fix the memory leak?",
+    "answer": "Replaced unbounded HashMap with LruCache cap=10k in src/cache.rs (PR #16)",
+    "goldSessionIds": ["sess-006"]
+  },
+  {
+    "id": "q-007",
+    "type": "single-session-api",
+    "question": "How did we handle the github API rate limit?",
+    "answer": "Conditional requests with If-None-Match etag and 304 caching via http-cache",
+    "goldSessionIds": ["sess-007"]
+  },
+  {
+    "id": "q-008",
+    "type": "single-session-db",
+    "question": "What was the schema migration approach for run_history?",
+    "answer": "Three-phase: nullable column + dual-write, backfill + flip reads, drop old column",
+    "goldSessionIds": ["sess-008"]
+  },
+  {
+    "id": "q-009",
+    "type": "single-session-infra",
+    "question": "How is the docs site deployed?",
+    "answer": "GitHub Actions docs.yml workflow + mdbook build + Cloudflare Pages on shipctl.dev",
+    "goldSessionIds": ["sess-009"]
+  },
+  {
+    "id": "q-010",
+    "type": "single-session-release",
+    "question": "Which PR set up the cross-platform release pipeline?",
+    "answer": "PR #19 with cross-rs for linux and native macos/windows builds",
+    "goldSessionIds": ["sess-010"]
+  },
+  {
+    "id": "q-011",
+    "type": "multi-session-causal",
+    "question": "What was the root cause of the staging incident, and where was it fixed?",
+    "answer": "SHIPCTL_TOKEN unset caused fallback to bad SC_TOKEN; fixed in PR #11 (sess-001) with precedence test; documented in post-mortem (sess-014)",
+    "goldSessionIds": ["sess-001", "sess-014"]
+  },
+  {
+    "id": "q-012",
+    "type": "preference",
+    "question": "Which async runtime does the team prefer for new code?",
+    "answer": "async-std (decided in arch review, not tokio)",
+    "goldSessionIds": ["sess-015"]
+  },
+  {
+    "id": "q-013",
+    "type": "preference",
+    "question": "What are the user's formatting preferences?",
+    "answer": "2-space indent in yaml, tab in rust, rust-analyzer over rls, cargo clippy before commit",
+    "goldSessionIds": ["sess-012"]
+  },
+  {
+    "id": "q-014",
+    "type": "multi-session-review",
+    "question": "What was the race condition Aria fixed in the schema migration?",
+    "answer": "Dual-write step had race when two writers hit same row; Aria added per-row advisory lock",
+    "goldSessionIds": ["sess-008", "sess-013"]
+  },
+  {
+    "id": "q-015",
+    "type": "temporal",
+    "question": "What was shipped on April 8th 2026?",
+    "answer": "Auth env-var precedence fix in PR #11, then started docker arm64 work",
+    "goldSessionIds": ["sess-001", "sess-011"]
+  }
+]
@@ -0,0 +1,77 @@
+[
+  {
+    "id": "sess-001",
+    "timestamp": "2026-04-08T09:14:00Z",
+    "content": "[user] auth check broken on staging when SHIPCTL_TOKEN unset, falling back to wrong env var\n[assistant] looked at src/auth.rs:42. precedence is SHIPCTL_TOKEN > SHIP_TOKEN > legacy SC_TOKEN. legacy path didn't validate. fixing precedence + adding length check.\n[user] yeah ship it\n[assistant] PR #11 opened with fix + regression test. covers all three env names."
+  },
+  {
+    "id": "sess-002",
+    "timestamp": "2026-04-08T14:30:00Z",
+    "content": "[user] docker build failing for arm64 on M2 runner\n[assistant] missing target in Dockerfile FROM line. need --platform=$BUILDPLATFORM on builder stage. updated Dockerfile + Makefile to set BUILDX_PLATFORMS=linux/amd64,linux/arm64. tested locally on apple silicon, builds clean."
+  },
+  {
+    "id": "sess-003",
+    "timestamp": "2026-04-09T10:02:00Z",
+    "content": "[user] retry logic is duplicated across handlers. consolidate.\n[assistant] extracted retry_with_backoff into src/retry.rs. exponential backoff base=200ms cap=30s jitter=full. all 4 callsites now use it. cuts ~80 lines."
+  },
+  {
+    "id": "sess-004",
+    "timestamp": "2026-04-10T11:45:00Z",
+    "content": "[user] add helm chart support\n[assistant] new subcommand `shipctl helm install`. wraps helm CLI, threads through --namespace and --values. PR #14 has the changes + 12 integration tests. Aria reviewed and approved."
+  },
+  {
+    "id": "sess-005",
+    "timestamp": "2026-04-11T16:20:00Z",
+    "content": "[user] test flake on macos-22 runner. emits_changekind_file_delete keeps failing\n[assistant] fsevents propagation slower on apple silicon, the 800ms wait isn't enough. bumped to 1500ms + describe-level retry: 2. all 15 fs-watcher tests pass now."
+  },
+  {
+    "id": "sess-006",
+    "timestamp": "2026-04-12T09:50:00Z",
+    "content": "[user] memory leak in long-running daemon. RSS grows 5MB/min.\n[assistant] traced to unbounded HashMap in src/cache.rs. swapped for LruCache with 10k cap. RSS stable at 180MB after 6h soak test. PR #16."
+  },
+  {
+    "id": "sess-007",
+    "timestamp": "2026-04-13T13:15:00Z",
+    "content": "[user] github API rate-limiting us when listing PRs. 5000/hr isn't enough during ci\n[assistant] added conditional requests with If-None-Match etag. cache 304 responses. drops actual API calls by ~70%. used http-cache crate."
+  },
+  {
+    "id": "sess-008",
+    "timestamp": "2026-04-15T10:30:00Z",
+    "content": "[user] schema migration for run_history table needs zero-downtime\n[assistant] three-phase migration. phase 1: add nullable new column + dual-write. phase 2: backfill + flip reads. phase 3: drop old column. avoids ALTER TABLE locks. ran on staging, took 4s for 2M rows."
+  },
+  {
+    "id": "sess-009",
+    "timestamp": "2026-04-16T15:40:00Z",
+    "content": "[user] docs site is stale. set up auto-deploy from main\n[assistant] added .github/workflows/docs.yml. mdbook build + cloudflare pages deploy. domain shipctl.dev wired up. preview deploys on PRs."
+  },
+  {
+    "id": "sess-010",
+    "timestamp": "2026-04-17T11:00:00Z",
+    "content": "[user] release pipeline. cut v0.3.0 with binaries for linux/macos/windows\n[assistant] new .github/workflows/release.yml. cross-rs for static linux, native for macos + windows. uploads to gh release + crates.io publish. tag-triggered. PR #19 has the workflow."
+  },
+  {
+    "id": "sess-011",
+    "timestamp": "2026-04-08T17:00:00Z",
+    "content": "[user] standup notes: shipped #11 auth fix this morning. starting on docker arm64 next."
+  },
+  {
+    "id": "sess-012",
+    "timestamp": "2026-04-10T18:30:00Z",
+    "content": "[user] preferences: I like rust analyzer over rls. always run cargo clippy before commit. prefer 2-space indent in yaml but tab in rust."
+  },
+  {
+    "id": "sess-013",
+    "timestamp": "2026-04-15T19:00:00Z",
+    "content": "[assistant] reviewed Aria's PR #18 (schema migration). flagged race condition in dual-write step when two writers hit same row. Aria added per-row advisory lock. lgtm now."
+  },
+  {
+    "id": "sess-014",
+    "timestamp": "2026-04-16T20:10:00Z",
+    "content": "[user] post-mortem from prod incident last week: SHIPCTL_TOKEN was unset in staging, fell back to bad SC_TOKEN which had wrong perms. delivery delayed 40min. action items: (1) precedence test (done in #11), (2) startup validation, (3) alert on auth fallback."
+  },
+  {
+    "id": "sess-015",
+    "timestamp": "2026-04-17T16:45:00Z",
+    "content": "[user] preferences: stick to async-std not tokio for new code. team agreed in arch review."
+  }
+]
@@ -0,0 +1,93 @@
+import type { Adapter, RankedDoc, Session } from "../types.js";
+
+interface AgentMemoryState {
+  baseUrl: string;
+  secret?: string;
+  sessions: Session[];
+  observationToSession: Map<string, string>;
+}
+
+interface RememberResponse {
+  memory?: { id?: string };
+  observationId?: string;
+  id?: string;
+  observation?: { id?: string };
+}
+
+interface SmartSearchResponse {
+  results?: Array<{
+    obsId?: string;
+    id?: string;
+    observationId?: string;
+    sessionId?: string;
+    score?: number;
+    content?: string;
+  }>;
+  observations?: Array<{
+    obsId?: string;
+    id?: string;
+    sessionId?: string;
+    score?: number;
+    content?: string;
+  }>;
+}
+
+function authHeaders(secret?: string): Record<string, string> {
+  const h: Record<string, string> = { "Content-Type": "application/json" };
+  if (secret) h.Authorization = `Bearer ${secret}`;
+  return h;
+}
+
+export const agentmemoryAdapter: Adapter<AgentMemoryState> = {
+  name: "agentmemory-hybrid",
+  async init(sessions, config) {
+    const baseUrl = (config?.baseUrl as string) ?? process.env.AGENTMEMORY_BASE_URL ?? "http://localhost:3111";
+    const secret = (config?.secret as string) ?? process.env.AGENTMEMORY_SECRET;
+    const observationToSession = new Map<string, string>();
+    for (const s of sessions) {
+      const res = await fetch(`${baseUrl}/agentmemory/remember`, {
+        method: "POST",
+        headers: authHeaders(secret),
+        body: JSON.stringify({
+          content: s.content,
+          type: "eval-session",
+          concepts: [s.id],
+        }),
+      });
+      if (!res.ok) {
+        throw new Error(`remember failed for ${s.id}: ${res.status} ${await res.text()}`);
+      }
+      const body = (await res.json()) as RememberResponse;
+      const obsId =
+        body.memory?.id ?? body.observationId ?? body.id ?? body.observation?.id;
+      if (obsId) observationToSession.set(obsId, s.id);
+    }
+    return { baseUrl, secret, sessions, observationToSession };
+  },
+  async query(q, state, k) {
+    const res = await fetch(`${state.baseUrl}/agentmemory/smart-search`, {
+      method: "POST",
+      headers: authHeaders(state.secret),
+      body: JSON.stringify({ query: q, limit: Math.max(k * 10, 50) }),
+    });
+    if (!res.ok) {
+      throw new Error(`smart-search failed: ${res.status} ${await res.text()}`);
+    }
+    const body = (await res.json()) as SmartSearchResponse;
+    const rows = body.results ?? body.observations ?? [];
+    const ranked: RankedDoc[] = [];
+    const seen = new Set<string>();
+    for (const row of rows) {
+      let sessionId = row.sessionId;
+      if (!sessionId) {
+        const memId = row.obsId ?? row.id ?? row.observationId;
+        sessionId = memId ? state.observationToSession.get(memId) : undefined;
+      }
+      if (!sessionId || seen.has(sessionId)) continue;
+      seen.add(sessionId);
+      ranked.push({ sessionId, score: row.score ?? 0 });
+      if (ranked.length >= k) break;
+    }
+    return ranked;
+  },
+};
@@ -0,0 +1,36 @@
+import type { Adapter, RankedDoc, Session } from "../types.js";
+
+interface GrepState {
+  sessions: Session[];
+}
+
+function tokenize(s: string): string[] {
+  return s
+    .toLowerCase()
+    .replace(/[^a-z0-9_]+/g, " ")
+    .split(/\s+/)
+    .filter((t) => t.length > 2);
+}
+
+export const grepAdapter: Adapter<GrepState> = {
+  name: "grep",
+  async init(sessions) {
+    return { sessions };
+  },
+  async query(q, state, k) {
+    const terms = tokenize(q);
+    const scored: RankedDoc[] = [];
+    for (const s of state.sessions) {
+      const body = s.content.toLowerCase();
+      let hits = 0;
+      for (const t of terms) {
+        if (body.includes(t)) hits += 1;
+      }
+      if (hits > 0) {
+        scored.push({ sessionId: s.id, score: hits });
+      }
+    }
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, k);
+  },
+};
@@ -0,0 +1,108 @@
+import type { Adapter, RankedDoc, Session } from "../types.js";
+
+interface VectorState {
+  sessions: Session[];
+  embeddings: Float32Array[];
+}
+
+const OPENAI_URL = "https://api.openai.com/v1/embeddings";
+const MODEL = "text-embedding-3-small";
+const DIM = 1536;
+
+async function embed(text: string, apiKey: string): Promise<Float32Array> {
+  const res = await fetch(OPENAI_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({ input: text, model: MODEL }),
+  });
+  if (!res.ok) {
+    throw new Error(`OpenAI embed failed: ${res.status} ${await res.text()}`);
+  }
+  const data = (await res.json()) as { data: Array<{ embedding: number[] }> };
+  return Float32Array.from(data.data[0].embedding);
+}
+
+async function embedBatch(texts: string[], apiKey: string): Promise<Float32Array[]> {
+  const res = await fetch(OPENAI_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({ input: texts, model: MODEL }),
+  });
+  if (!res.ok) {
+    throw new Error(`OpenAI batch embed failed: ${res.status} ${await res.text()}`);
+  }
+  const data = (await res.json()) as { data: Array<{ embedding: number[]; index: number }> };
+  if (!Array.isArray(data.data) || data.data.length !== texts.length) {
+    throw new Error(
+      `OpenAI batch embed: expected ${texts.length} embeddings, got ${data.data?.length ?? 0}`,
+    );
+  }
+  const out = new Array<Float32Array>(texts.length);
+  for (const row of data.data) {
+    if (
+      !Number.isInteger(row.index) ||
+      row.index < 0 ||
+      row.index >= texts.length ||
+      out[row.index] !== undefined
+    ) {
+      throw new Error(`OpenAI batch embed: invalid or duplicate index ${row.index}`);
+    }
+    if (!Array.isArray(row.embedding) || row.embedding.length === 0) {
+      throw new Error(`OpenAI batch embed: empty embedding at index ${row.index}`);
+    }
+    out[row.index] = Float32Array.from(row.embedding);
+  }
+  return out;
+}
+
+function cosine(a: Float32Array, b: Float32Array): number {
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+export const vectorAdapter: Adapter<VectorState> = {
+  name: "vector",
+  async init(sessions) {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) throw new Error("OPENAI_API_KEY required for vector adapter");
+    const embeddings: Float32Array[] = new Array(sessions.length);
+    const BATCH = 50;
+    for (let i = 0; i < sessions.length; i += BATCH) {
+      const batch = sessions.slice(i, i + BATCH);
+      const vecs = await embedBatch(
+        batch.map((s) => s.content.slice(0, 8000)),
+        apiKey,
+      );
+      for (let j = 0; j < vecs.length; j++) embeddings[i + j] = vecs[j];
+    }
+    if (embeddings.length > 0 && embeddings[0].length !== DIM) {
+      throw new Error(`unexpected embedding dim: ${embeddings[0].length}`);
+    }
+    return { sessions, embeddings };
+  },
+  async query(q, state, k) {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) throw new Error("OPENAI_API_KEY required for vector adapter");
+    const qvec = await embed(q, apiKey);
+    const scored: RankedDoc[] = state.sessions.map((s, i) => ({
+      sessionId: s.id,
+      score: cosine(qvec, state.embeddings[i]),
+    }));
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, k);
+  },
+};
@@ -0,0 +1,101 @@
+import { readFileSync, existsSync, mkdirSync, writeFileSync, appendFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { agentmemoryAdapter } from "./adapters/agentmemory.js";
+import { grepAdapter } from "./adapters/grep.js";
+import { vectorAdapter } from "./adapters/vector.js";
+import { aggregate, scoreQuestion } from "./score.js";
+import type { Adapter, Question, ScoreRow, Session } from "./types.js";
+
+const ADAPTERS: Record<string, Adapter> = {
+  grep: grepAdapter as unknown as Adapter,
+  vector: vectorAdapter as unknown as Adapter,
+  agentmemory: agentmemoryAdapter as unknown as Adapter,
+};
+
+interface CliOptions {
+  data: string;
+  adapters: string;
+  k: string;
+  out: string;
+}
+
+function parse(): CliOptions {
+  const { values } = parseArgs({
+    options: {
+      data: { type: "string", default: "eval/data/coding-agent-life-v1" },
+      adapters: { type: "string", default: "grep,vector,agentmemory" },
+      k: { type: "string", default: "5" },
+      out: { type: "string", default: "eval/reports/coding-life" },
+    },
+  });
+  return values as unknown as CliOptions;
+}
+
+async function main(): Promise<void> {
+  const opts = parse();
+  const k = Number(opts.k);
+  if (!Number.isInteger(k) || k <= 0) {
+    console.error(`--k must be a positive integer, got: ${opts.k}`);
+    process.exit(2);
+  }
+  const sessions = JSON.parse(
+    readFileSync(resolve(opts.data, "sessions.json"), "utf8"),
+  ) as Session[];
+  const queriesRaw = JSON.parse(
+    readFileSync(resolve(opts.data, "queries.json"), "utf8"),
+  ) as Array<Omit<Question, "haystack">>;
+  const questions: Question[] = queriesRaw.map((q) => ({ ...q, haystack: sessions }));
+  const adapterNames = opts.adapters.split(",").map((s) => s.trim()).filter(Boolean);
+  for (const a of adapterNames) {
+    if (!ADAPTERS[a]) {
+      console.error(`unknown adapter: ${a}. options: ${Object.keys(ADAPTERS).join(",")}`);
+      process.exit(2);
+    }
+  }
+  console.log(
+    `loaded ${sessions.length} sessions, ${questions.length} queries, adapters: ${adapterNames.join(",")}, k=${k}`,
+  );
+
+  const outDir = resolve(opts.out);
+  mkdirSync(outDir, { recursive: true });
+  const ndjsonPath = `${outDir}/scores.ndjson`;
+  if (existsSync(ndjsonPath)) writeFileSync(ndjsonPath, "");
+
+  const rows: ScoreRow[] = [];
+  for (const adapterName of adapterNames) {
+    const adapter = ADAPTERS[adapterName];
+    console.log(`\n== ${adapter.name} ==`);
+    const state = await adapter.init(sessions);
+    try {
+      for (const q of questions) {
+        const t0 = performance.now();
+        const ranked = await adapter.query(q.question, state, k);
+        const latencyMs = performance.now() - t0;
+        const row = scoreQuestion(q, ranked, k, adapter.name, latencyMs);
+        rows.push(row);
+        appendFileSync(ndjsonPath, JSON.stringify(row) + "\n");
+        const mark = row.hit ? "+" : "-";
+        console.log(
+          `  ${mark} ${q.id} [${q.type}] R@${k}=${row.recallAtK.toFixed(2)} (${Math.round(latencyMs)}ms)`,
+        );
+      }
+    } finally {
+      if (adapter.teardown) await adapter.teardown(state);
+    }
+  }
+
+  const agg = aggregate(rows);
+  writeFileSync(`${outDir}/summary.json`, JSON.stringify(agg, null, 2));
+  console.log("\n=== Summary ===");
+  for (const [adapter, stats] of Object.entries(agg.byAdapter)) {
+    console.log(
+      `  ${adapter.padEnd(22)} P@${k}=${stats.p.toFixed(3)} R@${k}=${stats.r.toFixed(3)} hit=${stats.hit}/${stats.n} p50=${Math.round(stats.latencyP50)}ms`,
+    );
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
@@ -0,0 +1,54 @@
+import { readFileSync } from "node:fs";
+import type { Question, Session } from "./types.js";
+
+interface LongMemEvalRaw {
+  question_id: string;
+  question_type: string;
+  question: string;
+  answer?: string;
+  answer_session_ids: string[];
+  haystack_session_ids: string[];
+  haystack_sessions: Array<Array<{ role: string; content: string }>>;
+}
+
+function flattenSession(turns: Array<{ role: string; content: string }>): string {
+  return turns.map((t) => `[${t.role}] ${t.content}`).join("\n\n");
+}
+
+export function loadLongMemEval(path: string, limit?: number): Question[] {
+  const raw = JSON.parse(readFileSync(path, "utf8")) as LongMemEvalRaw[];
+  const slice = typeof limit === "number" ? raw.slice(0, limit) : raw;
+  const questions: Question[] = [];
+  for (const r of slice) {
+    if (r.haystack_session_ids.length !== r.haystack_sessions.length) {
+      throw new Error(
+        `LongMemEval row ${r.question_id}: haystack_session_ids (${r.haystack_session_ids.length}) and haystack_sessions (${r.haystack_sessions.length}) length mismatch`,
+      );
+    }
+    const haystack: Session[] = r.haystack_session_ids.map((id, i) => ({
+      id,
+      content: flattenSession(r.haystack_sessions[i]),
+    }));
+    questions.push({
+      id: r.question_id,
+      type: r.question_type,
+      question: r.question,
+      answer: r.answer,
+      goldSessionIds: r.answer_session_ids,
+      haystack,
+    });
+  }
+  return questions;
+}
+
+export function stratifySample(questions: Question[], perType: number): Question[] {
+  const buckets: Record<string, Question[]> = {};
+  for (const q of questions) {
+    (buckets[q.type] ??= []).push(q);
+  }
+  const out: Question[] = [];
+  for (const type of Object.keys(buckets).sort()) {
+    out.push(...buckets[type].slice(0, perType));
+  }
+  return out;
+}
@@ -0,0 +1,126 @@
+import { existsSync, mkdirSync, writeFileSync, appendFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { agentmemoryAdapter } from "./adapters/agentmemory.js";
+import { grepAdapter } from "./adapters/grep.js";
+import { vectorAdapter } from "./adapters/vector.js";
+import { loadLongMemEval, stratifySample } from "./load.js";
+import { aggregate, scoreQuestion } from "./score.js";
+import type { Adapter, ScoreRow } from "./types.js";
+
+const ADAPTERS: Record<string, Adapter> = {
+  grep: grepAdapter as unknown as Adapter,
+  vector: vectorAdapter as unknown as Adapter,
+  agentmemory: agentmemoryAdapter as unknown as Adapter,
+};
+
+interface CliOptions {
+  data: string;
+  adapters: string;
+  k: string;
+  limit?: string;
+  stratify?: string;
+  out: string;
+}
+
+function parse(): CliOptions {
+  const { values } = parseArgs({
+    options: {
+      data: { type: "string", default: process.env.LONGMEMEVAL_PATH ?? "" },
+      adapters: { type: "string", default: "grep,vector,agentmemory" },
+      k: { type: "string", default: "5" },
+      limit: { type: "string" },
+      stratify: { type: "string" },
+      out: { type: "string", default: "eval/reports/longmemeval" },
+    },
+  });
+  return values as unknown as CliOptions;
+}
+
+async function main(): Promise<void> {
+  const opts = parse();
+  if (!opts.data) {
+    console.error("--data <path/to/longmemeval_s.json> required (or LONGMEMEVAL_PATH env)");
+    process.exit(2);
+  }
+  const k = Number(opts.k);
+  if (!Number.isInteger(k) || k <= 0) {
+    console.error(`--k must be a positive integer, got: ${opts.k}`);
+    process.exit(2);
+  }
+  let limit: number | undefined;
+  if (opts.limit !== undefined) {
+    limit = Number(opts.limit);
+    if (!Number.isInteger(limit) || limit <= 0) {
+      console.error(`--limit must be a positive integer, got: ${opts.limit}`);
+      process.exit(2);
+    }
+  }
+  let perType: number | undefined;
+  if (opts.stratify !== undefined) {
+    perType = Number(opts.stratify);
+    if (!Number.isInteger(perType) || perType <= 0) {
+      console.error(`--stratify must be a positive integer, got: ${opts.stratify}`);
+      process.exit(2);
+    }
+  }
+  const adapterNames = opts.adapters.split(",").map((s) => s.trim()).filter(Boolean);
+  for (const a of adapterNames) {
+    if (!ADAPTERS[a]) {
+      console.error(`unknown adapter: ${a}. options: ${Object.keys(ADAPTERS).join(",")}`);
+      process.exit(2);
+    }
+  }
+  let questions = loadLongMemEval(resolve(opts.data), limit);
+  if (perType) questions = stratifySample(questions, perType);
+  console.log(
+    `loaded ${questions.length} questions, adapters: ${adapterNames.join(",")}, k=${k}`,
+  );
+
+  const outDir = resolve(opts.out);
+  mkdirSync(outDir, { recursive: true });
+  const ndjsonPath = `${outDir}/scores.ndjson`;
+  if (existsSync(ndjsonPath)) writeFileSync(ndjsonPath, "");
+  mkdirSync(dirname(ndjsonPath), { recursive: true });
+
+  const rows: ScoreRow[] = [];
+  for (const adapterName of adapterNames) {
+    const adapter = ADAPTERS[adapterName];
+    console.log(`\n== ${adapter.name} ==`);
+    for (const q of questions) {
+      const t0 = performance.now();
+      const state = await adapter.init(q.haystack);
+      try {
+        const ranked = await adapter.query(q.question, state, k);
+        const latencyMs = performance.now() - t0;
+        const row = scoreQuestion(q, ranked, k, adapter.name, latencyMs);
+        rows.push(row);
+        appendFileSync(ndjsonPath, JSON.stringify(row) + "\n");
+        const mark = row.hit ? "+" : "-";
+        console.log(
+          `  ${mark} ${q.id} [${q.type}] R@${k}=${row.recallAtK.toFixed(2)} (${Math.round(latencyMs)}ms)`,
+        );
+      } finally {
+        if (adapter.teardown) await adapter.teardown(state);
+      }
+    }
+  }
+
+  const agg = aggregate(rows);
+  const summaryPath = `${outDir}/summary.json`;
+  writeFileSync(summaryPath, JSON.stringify(agg, null, 2));
+
+  console.log("\n=== Summary ===");
+  for (const [adapter, stats] of Object.entries(agg.byAdapter)) {
+    console.log(
+      `  ${adapter.padEnd(22)} P@${k}=${stats.p.toFixed(3)} R@${k}=${stats.r.toFixed(3)} hit=${stats.hit}/${stats.n} p50=${Math.round(stats.latencyP50)}ms`,
+    );
+  }
+  console.log(`\nwrote ${ndjsonPath}`);
+  console.log(`wrote ${summaryPath}`);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
@@ -0,0 +1,78 @@
+import type { Question, RankedDoc, ScoreRow } from "./types.js";
+
+export function scoreQuestion(
+  q: Question,
+  ranked: RankedDoc[],
+  k: number,
+  adapter: string,
+  latencyMs: number,
+): ScoreRow {
+  const topK = ranked.slice(0, k).map((r) => r.sessionId);
+  const gold = new Set(q.goldSessionIds);
+  const hits = topK.filter((id) => gold.has(id)).length;
+  const precisionAtK = k > 0 ? hits / k : 0;
+  const recallAtK = gold.size === 0 ? 0 : hits / gold.size;
+  const hit = hits > 0;
+  let topGoldRank: number | null = null;
+  for (let i = 0; i < ranked.length; i++) {
+    if (gold.has(ranked[i].sessionId)) {
+      topGoldRank = i + 1;
+      break;
+    }
+  }
+  return {
+    questionId: q.id,
+    questionType: q.type,
+    adapter,
+    k,
+    precisionAtK,
+    recallAtK,
+    hit,
+    topGoldRank,
+    latencyMs,
+  };
+}
+
+export function aggregate(rows: ScoreRow[]): {
+  byAdapter: Record<string, { p: number; r: number; hit: number; n: number; latencyP50: number }>;
+  byType: Record<string, Record<string, { p: number; r: number; hit: number; n: number }>>;
+} {
+  const byAdapter: Record<
+    string,
+    { p: number; r: number; hit: number; n: number; latencyP50: number }
+  > = {};
+  const latencies: Record<string, number[]> = {};
+  for (const r of rows) {
+    const a = (byAdapter[r.adapter] ??= { p: 0, r: 0, hit: 0, n: 0, latencyP50: 0 });
+    a.p += r.precisionAtK;
+    a.r += r.recallAtK;
+    a.hit += r.hit ? 1 : 0;
+    a.n += 1;
+    (latencies[r.adapter] ??= []).push(r.latencyMs);
+  }
+  for (const adapter of Object.keys(byAdapter)) {
+    const a = byAdapter[adapter];
+    a.p = a.p / a.n;
+    a.r = a.r / a.n;
+    const sorted = latencies[adapter].slice().sort((x, y) => x - y);
+    a.latencyP50 = sorted[Math.floor(sorted.length / 2)] ?? 0;
+  }
+  const byType: Record<string, Record<string, { p: number; r: number; hit: number; n: number }>> =
+    {};
+  for (const r of rows) {
+    const t = (byType[r.questionType] ??= {});
+    const a = (t[r.adapter] ??= { p: 0, r: 0, hit: 0, n: 0 });
+    a.p += r.precisionAtK;
+    a.r += r.recallAtK;
+    a.hit += r.hit ? 1 : 0;
+    a.n += 1;
+  }
+  for (const t of Object.keys(byType)) {
+    for (const adapter of Object.keys(byType[t])) {
+      const a = byType[t][adapter];
+      a.p = a.p / a.n;
+      a.r = a.r / a.n;
+    }
+  }
+  return { byAdapter, byType };
+}
@@ -0,0 +1,38 @@
+export interface Session {
+  id: string;
+  timestamp?: string;
+  content: string;
+}
+
+export interface Question {
+  id: string;
+  type: string;
+  question: string;
+  answer?: string;
+  goldSessionIds: string[];
+  haystack: Session[];
+}
+
+export interface RankedDoc {
+  sessionId: string;
+  score: number;
+}
+
+export interface Adapter<State = unknown> {
+  name: string;
+  init(sessions: Session[], config?: Record<string, unknown>): Promise<State>;
+  query(q: string, state: State, k: number): Promise<RankedDoc[]>;
+  teardown?(state: State): Promise<void>;
+}
+
+export interface ScoreRow {
+  questionId: string;
+  questionType: string;
+  adapter: string;
+  k: number;
+  precisionAtK: number;
+  recallAtK: number;
+  hit: boolean;
+  topGoldRank: number | null;
+  latencyMs: number;
+}
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+# Boot a sandboxed agentmemory + iii-engine on alt ports with a clean data dir,
+# so eval runs aren't polluted by (and don't pollute) your real ~/.agentmemory.
+# Source it: `source eval/scripts/sandbox.sh` then run eval scripts;
+# the sandbox is torn down on EXIT.
+
+set -euo pipefail
+
+SANDBOX_ROOT="${SANDBOX_ROOT:-/tmp/agentmemory-eval-sandbox}"
+SANDBOX_PORT="${SANDBOX_PORT:-3411}"
+SANDBOX_STREAM_PORT="${SANDBOX_STREAM_PORT:-3412}"
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+if ! command -v iii >/dev/null 2>&1; then
+  echo "iii binary not on PATH. Install pinned version:"
+  echo "  curl -fsSL https://github.com/iii-hq/iii/releases/download/iii/v0.11.2/iii-aarch64-apple-darwin.tar.gz | tar -xz -C ~/.local/bin"
+  exit 1
+fi
+
+iii_ver=$(iii --version 2>&1 | head -1)
+if [[ "$iii_ver" != "0.11.2" ]]; then
+  echo "warning: iii version on PATH is $iii_ver; agentmemory pins 0.11.2"
+fi
+
+if [[ ! -f "$REPO_ROOT/dist/index.mjs" ]]; then
+  echo "dist/ missing. Run: npm run build" >&2
+  exit 1
+fi
+
+if [[ -z "${SANDBOX_ROOT:-}" || "$SANDBOX_ROOT" == "/" || "$SANDBOX_ROOT" != /tmp/* ]]; then
+  echo "refusing to wipe SANDBOX_ROOT='$SANDBOX_ROOT' — must be non-empty and under /tmp/" >&2
+  exit 1
+fi
+rm -rf "$SANDBOX_ROOT"
+mkdir -p "$SANDBOX_ROOT/data" "$SANDBOX_ROOT/.agentmemory"
+
+cat > "$SANDBOX_ROOT/iii-config.yaml" <<EOF
+workers:
+  - name: iii-http
+    config:
+      port: $SANDBOX_PORT
+      host: 127.0.0.1
+      default_timeout: 180000
+      cors:
+        allowed_origins: ["http://localhost:$SANDBOX_PORT", "http://127.0.0.1:$SANDBOX_PORT"]
+        allowed_methods: [GET, POST, PUT, DELETE, OPTIONS]
+  - name: iii-state
+    config:
+      adapter:
+        name: kv
+        config:
+          store_method: file_based
+          file_path: $SANDBOX_ROOT/data/state_store.db
+  - name: iii-queue
+    config:
+      adapter:
+        name: builtin
+  - name: iii-pubsub
+    config:
+      adapter:
+        name: local
+  - name: iii-cron
+    config:
+      adapter:
+        name: kv
+  - name: iii-stream
+    config:
+      port: $SANDBOX_STREAM_PORT
+      host: 127.0.0.1
+      adapter:
+        name: kv
+        config:
+          store_method: file_based
+          file_path: $SANDBOX_ROOT/data/stream_store
+  - name: iii-observability
+    config:
+      enabled: true
+      service_name: agentmemory-eval
+      exporter: memory
+      sampling_ratio: 1.0
+      metrics_enabled: true
+      logs_enabled: false
+      logs_console_output: false
+  - name: iii-exec
+    config:
+      exec:
+        - node $REPO_ROOT/dist/index.mjs
+EOF
+
+cd "$SANDBOX_ROOT"
+HOME="$SANDBOX_ROOT" iii --config "$SANDBOX_ROOT/iii-config.yaml" > "$SANDBOX_ROOT/iii.log" 2>&1 &
+SANDBOX_PID=$!
+
+cleanup() {
+  echo "tearing down sandbox (pid $SANDBOX_PID)"
+  kill "$SANDBOX_PID" 2>/dev/null || true
+  sleep 1
+  kill -9 "$SANDBOX_PID" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# wait for livez
+for i in $(seq 1 30); do
+  if curl -sS --max-time 1 "http://localhost:$SANDBOX_PORT/agentmemory/livez" 2>/dev/null | grep -q '"status":"ok"'; then
+    export AGENTMEMORY_BASE_URL="http://localhost:$SANDBOX_PORT"
+    echo "sandbox ready: $AGENTMEMORY_BASE_URL"
+    echo "  state: $SANDBOX_ROOT/data/"
+    echo "  logs:  $SANDBOX_ROOT/iii.log"
+    return 0 2>/dev/null || exit 0
+  fi
+  sleep 1
+done
+
+echo "sandbox failed to come up within 30s. last log lines:" >&2
+tail -10 "$SANDBOX_ROOT/iii.log" >&2
+exit 1