chore: import upstream snapshot with attribution

eval/runner/adapters/agentmemory.ts

@@ -0,0 +1,93 @@
+import type { Adapter, RankedDoc, Session } from "../types.js";
+
+interface AgentMemoryState {
+  baseUrl: string;
+  secret?: string;
+  sessions: Session[];
+  observationToSession: Map<string, string>;
+}
+
+interface RememberResponse {
+  memory?: { id?: string };
+  observationId?: string;
+  id?: string;
+  observation?: { id?: string };
+}
+
+interface SmartSearchResponse {
+  results?: Array<{
+    obsId?: string;
+    id?: string;
+    observationId?: string;
+    sessionId?: string;
+    score?: number;
+    content?: string;
+  }>;
+  observations?: Array<{
+    obsId?: string;
+    id?: string;
+    sessionId?: string;
+    score?: number;
+    content?: string;
+  }>;
+}
+
+function authHeaders(secret?: string): Record<string, string> {
+  const h: Record<string, string> = { "Content-Type": "application/json" };
+  if (secret) h.Authorization = `Bearer ${secret}`;
+  return h;
+}
+
+export const agentmemoryAdapter: Adapter<AgentMemoryState> = {
+  name: "agentmemory-hybrid",
+  async init(sessions, config) {
+    const baseUrl = (config?.baseUrl as string) ?? process.env.AGENTMEMORY_BASE_URL ?? "http://localhost:3111";
+    const secret = (config?.secret as string) ?? process.env.AGENTMEMORY_SECRET;
+    const observationToSession = new Map<string, string>();
+    for (const s of sessions) {
+      const res = await fetch(`${baseUrl}/agentmemory/remember`, {
+        method: "POST",
+        headers: authHeaders(secret),
+        body: JSON.stringify({
+          content: s.content,
+          type: "eval-session",
+          concepts: [s.id],
+        }),
+      });
+      if (!res.ok) {
+        throw new Error(`remember failed for ${s.id}: ${res.status} ${await res.text()}`);
+      }
+      const body = (await res.json()) as RememberResponse;
+      const obsId =
+        body.memory?.id ?? body.observationId ?? body.id ?? body.observation?.id;
+      if (obsId) observationToSession.set(obsId, s.id);
+    }
+    return { baseUrl, secret, sessions, observationToSession };
+  },
+  async query(q, state, k) {
+    const res = await fetch(`${state.baseUrl}/agentmemory/smart-search`, {
+      method: "POST",
+      headers: authHeaders(state.secret),
+      body: JSON.stringify({ query: q, limit: Math.max(k * 10, 50) }),
+    });
+    if (!res.ok) {
+      throw new Error(`smart-search failed: ${res.status} ${await res.text()}`);
+    }
+    const body = (await res.json()) as SmartSearchResponse;
+    const rows = body.results ?? body.observations ?? [];
+    const ranked: RankedDoc[] = [];
+    const seen = new Set<string>();
+    for (const row of rows) {
+      let sessionId = row.sessionId;
+      if (!sessionId) {
+        const memId = row.obsId ?? row.id ?? row.observationId;
+        sessionId = memId ? state.observationToSession.get(memId) : undefined;
+      }
+      if (!sessionId || seen.has(sessionId)) continue;
+      seen.add(sessionId);
+      ranked.push({ sessionId, score: row.score ?? 0 });
+      if (ranked.length >= k) break;
+    }
+    return ranked;
+  },
+};

eval/runner/adapters/grep.ts

@@ -0,0 +1,36 @@
+import type { Adapter, RankedDoc, Session } from "../types.js";
+
+interface GrepState {
+  sessions: Session[];
+}
+
+function tokenize(s: string): string[] {
+  return s
+    .toLowerCase()
+    .replace(/[^a-z0-9_]+/g, " ")
+    .split(/\s+/)
+    .filter((t) => t.length > 2);
+}
+
+export const grepAdapter: Adapter<GrepState> = {
+  name: "grep",
+  async init(sessions) {
+    return { sessions };
+  },
+  async query(q, state, k) {
+    const terms = tokenize(q);
+    const scored: RankedDoc[] = [];
+    for (const s of state.sessions) {
+      const body = s.content.toLowerCase();
+      let hits = 0;
+      for (const t of terms) {
+        if (body.includes(t)) hits += 1;
+      }
+      if (hits > 0) {
+        scored.push({ sessionId: s.id, score: hits });
+      }
+    }
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, k);
+  },
+};

eval/runner/adapters/vector.ts

@@ -0,0 +1,108 @@
+import type { Adapter, RankedDoc, Session } from "../types.js";
+
+interface VectorState {
+  sessions: Session[];
+  embeddings: Float32Array[];
+}
+
+const OPENAI_URL = "https://api.openai.com/v1/embeddings";
+const MODEL = "text-embedding-3-small";
+const DIM = 1536;
+
+async function embed(text: string, apiKey: string): Promise<Float32Array> {
+  const res = await fetch(OPENAI_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({ input: text, model: MODEL }),
+  });
+  if (!res.ok) {
+    throw new Error(`OpenAI embed failed: ${res.status} ${await res.text()}`);
+  }
+  const data = (await res.json()) as { data: Array<{ embedding: number[] }> };
+  return Float32Array.from(data.data[0].embedding);
+}
+
+async function embedBatch(texts: string[], apiKey: string): Promise<Float32Array[]> {
+  const res = await fetch(OPENAI_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({ input: texts, model: MODEL }),
+  });
+  if (!res.ok) {
+    throw new Error(`OpenAI batch embed failed: ${res.status} ${await res.text()}`);
+  }
+  const data = (await res.json()) as { data: Array<{ embedding: number[]; index: number }> };
+  if (!Array.isArray(data.data) || data.data.length !== texts.length) {
+    throw new Error(
+      `OpenAI batch embed: expected ${texts.length} embeddings, got ${data.data?.length ?? 0}`,
+    );
+  }
+  const out = new Array<Float32Array>(texts.length);
+  for (const row of data.data) {
+    if (
+      !Number.isInteger(row.index) ||
+      row.index < 0 ||
+      row.index >= texts.length ||
+      out[row.index] !== undefined
+    ) {
+      throw new Error(`OpenAI batch embed: invalid or duplicate index ${row.index}`);
+    }
+    if (!Array.isArray(row.embedding) || row.embedding.length === 0) {
+      throw new Error(`OpenAI batch embed: empty embedding at index ${row.index}`);
+    }
+    out[row.index] = Float32Array.from(row.embedding);
+  }
+  return out;
+}
+
+function cosine(a: Float32Array, b: Float32Array): number {
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+export const vectorAdapter: Adapter<VectorState> = {
+  name: "vector",
+  async init(sessions) {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) throw new Error("OPENAI_API_KEY required for vector adapter");
+    const embeddings: Float32Array[] = new Array(sessions.length);
+    const BATCH = 50;
+    for (let i = 0; i < sessions.length; i += BATCH) {
+      const batch = sessions.slice(i, i + BATCH);
+      const vecs = await embedBatch(
+        batch.map((s) => s.content.slice(0, 8000)),
+        apiKey,
+      );
+      for (let j = 0; j < vecs.length; j++) embeddings[i + j] = vecs[j];
+    }
+    if (embeddings.length > 0 && embeddings[0].length !== DIM) {
+      throw new Error(`unexpected embedding dim: ${embeddings[0].length}`);
+    }
+    return { sessions, embeddings };
+  },
+  async query(q, state, k) {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) throw new Error("OPENAI_API_KEY required for vector adapter");
+    const qvec = await embed(q, apiKey);
+    const scored: RankedDoc[] = state.sessions.map((s, i) => ({
+      sessionId: s.id,
+      score: cosine(qvec, state.embeddings[i]),
+    }));
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, k);
+  },
+};

eval/runner/coding-life.ts

@@ -0,0 +1,101 @@
+import { readFileSync, existsSync, mkdirSync, writeFileSync, appendFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { agentmemoryAdapter } from "./adapters/agentmemory.js";
+import { grepAdapter } from "./adapters/grep.js";
+import { vectorAdapter } from "./adapters/vector.js";
+import { aggregate, scoreQuestion } from "./score.js";
+import type { Adapter, Question, ScoreRow, Session } from "./types.js";
+
+const ADAPTERS: Record<string, Adapter> = {
+  grep: grepAdapter as unknown as Adapter,
+  vector: vectorAdapter as unknown as Adapter,
+  agentmemory: agentmemoryAdapter as unknown as Adapter,
+};
+
+interface CliOptions {
+  data: string;
+  adapters: string;
+  k: string;
+  out: string;
+}
+
+function parse(): CliOptions {
+  const { values } = parseArgs({
+    options: {
+      data: { type: "string", default: "eval/data/coding-agent-life-v1" },
+      adapters: { type: "string", default: "grep,vector,agentmemory" },
+      k: { type: "string", default: "5" },
+      out: { type: "string", default: "eval/reports/coding-life" },
+    },
+  });
+  return values as unknown as CliOptions;
+}
+
+async function main(): Promise<void> {
+  const opts = parse();
+  const k = Number(opts.k);
+  if (!Number.isInteger(k) || k <= 0) {
+    console.error(`--k must be a positive integer, got: ${opts.k}`);
+    process.exit(2);
+  }
+  const sessions = JSON.parse(
+    readFileSync(resolve(opts.data, "sessions.json"), "utf8"),
+  ) as Session[];
+  const queriesRaw = JSON.parse(
+    readFileSync(resolve(opts.data, "queries.json"), "utf8"),
+  ) as Array<Omit<Question, "haystack">>;
+  const questions: Question[] = queriesRaw.map((q) => ({ ...q, haystack: sessions }));
+  const adapterNames = opts.adapters.split(",").map((s) => s.trim()).filter(Boolean);
+  for (const a of adapterNames) {
+    if (!ADAPTERS[a]) {
+      console.error(`unknown adapter: ${a}. options: ${Object.keys(ADAPTERS).join(",")}`);
+      process.exit(2);
+    }
+  }
+  console.log(
+    `loaded ${sessions.length} sessions, ${questions.length} queries, adapters: ${adapterNames.join(",")}, k=${k}`,
+  );
+
+  const outDir = resolve(opts.out);
+  mkdirSync(outDir, { recursive: true });
+  const ndjsonPath = `${outDir}/scores.ndjson`;
+  if (existsSync(ndjsonPath)) writeFileSync(ndjsonPath, "");
+
+  const rows: ScoreRow[] = [];
+  for (const adapterName of adapterNames) {
+    const adapter = ADAPTERS[adapterName];
+    console.log(`\n== ${adapter.name} ==`);
+    const state = await adapter.init(sessions);
+    try {
+      for (const q of questions) {
+        const t0 = performance.now();
+        const ranked = await adapter.query(q.question, state, k);
+        const latencyMs = performance.now() - t0;
+        const row = scoreQuestion(q, ranked, k, adapter.name, latencyMs);
+        rows.push(row);
+        appendFileSync(ndjsonPath, JSON.stringify(row) + "\n");
+        const mark = row.hit ? "+" : "-";
+        console.log(
+          `  ${mark} ${q.id} [${q.type}] R@${k}=${row.recallAtK.toFixed(2)} (${Math.round(latencyMs)}ms)`,
+        );
+      }
+    } finally {
+      if (adapter.teardown) await adapter.teardown(state);
+    }
+  }
+
+  const agg = aggregate(rows);
+  writeFileSync(`${outDir}/summary.json`, JSON.stringify(agg, null, 2));
+  console.log("\n=== Summary ===");
+  for (const [adapter, stats] of Object.entries(agg.byAdapter)) {
+    console.log(
+      `  ${adapter.padEnd(22)} P@${k}=${stats.p.toFixed(3)} R@${k}=${stats.r.toFixed(3)} hit=${stats.hit}/${stats.n} p50=${Math.round(stats.latencyP50)}ms`,
+    );
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

eval/runner/load.ts

@@ -0,0 +1,54 @@
+import { readFileSync } from "node:fs";
+import type { Question, Session } from "./types.js";
+
+interface LongMemEvalRaw {
+  question_id: string;
+  question_type: string;
+  question: string;
+  answer?: string;
+  answer_session_ids: string[];
+  haystack_session_ids: string[];
+  haystack_sessions: Array<Array<{ role: string; content: string }>>;
+}
+
+function flattenSession(turns: Array<{ role: string; content: string }>): string {
+  return turns.map((t) => `[${t.role}] ${t.content}`).join("\n\n");
+}
+
+export function loadLongMemEval(path: string, limit?: number): Question[] {
+  const raw = JSON.parse(readFileSync(path, "utf8")) as LongMemEvalRaw[];
+  const slice = typeof limit === "number" ? raw.slice(0, limit) : raw;
+  const questions: Question[] = [];
+  for (const r of slice) {
+    if (r.haystack_session_ids.length !== r.haystack_sessions.length) {
+      throw new Error(
+        `LongMemEval row ${r.question_id}: haystack_session_ids (${r.haystack_session_ids.length}) and haystack_sessions (${r.haystack_sessions.length}) length mismatch`,
+      );
+    }
+    const haystack: Session[] = r.haystack_session_ids.map((id, i) => ({
+      id,
+      content: flattenSession(r.haystack_sessions[i]),
+    }));
+    questions.push({
+      id: r.question_id,
+      type: r.question_type,
+      question: r.question,
+      answer: r.answer,
+      goldSessionIds: r.answer_session_ids,
+      haystack,
+    });
+  }
+  return questions;
+}
+
+export function stratifySample(questions: Question[], perType: number): Question[] {
+  const buckets: Record<string, Question[]> = {};
+  for (const q of questions) {
+    (buckets[q.type] ??= []).push(q);
+  }
+  const out: Question[] = [];
+  for (const type of Object.keys(buckets).sort()) {
+    out.push(...buckets[type].slice(0, perType));
+  }
+  return out;
+}

eval/runner/longmemeval.ts

@@ -0,0 +1,126 @@
+import { existsSync, mkdirSync, writeFileSync, appendFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { agentmemoryAdapter } from "./adapters/agentmemory.js";
+import { grepAdapter } from "./adapters/grep.js";
+import { vectorAdapter } from "./adapters/vector.js";
+import { loadLongMemEval, stratifySample } from "./load.js";
+import { aggregate, scoreQuestion } from "./score.js";
+import type { Adapter, ScoreRow } from "./types.js";
+
+const ADAPTERS: Record<string, Adapter> = {
+  grep: grepAdapter as unknown as Adapter,
+  vector: vectorAdapter as unknown as Adapter,
+  agentmemory: agentmemoryAdapter as unknown as Adapter,
+};
+
+interface CliOptions {
+  data: string;
+  adapters: string;
+  k: string;
+  limit?: string;
+  stratify?: string;
+  out: string;
+}
+
+function parse(): CliOptions {
+  const { values } = parseArgs({
+    options: {
+      data: { type: "string", default: process.env.LONGMEMEVAL_PATH ?? "" },
+      adapters: { type: "string", default: "grep,vector,agentmemory" },
+      k: { type: "string", default: "5" },
+      limit: { type: "string" },
+      stratify: { type: "string" },
+      out: { type: "string", default: "eval/reports/longmemeval" },
+    },
+  });
+  return values as unknown as CliOptions;
+}
+
+async function main(): Promise<void> {
+  const opts = parse();
+  if (!opts.data) {
+    console.error("--data <path/to/longmemeval_s.json> required (or LONGMEMEVAL_PATH env)");
+    process.exit(2);
+  }
+  const k = Number(opts.k);
+  if (!Number.isInteger(k) || k <= 0) {
+    console.error(`--k must be a positive integer, got: ${opts.k}`);
+    process.exit(2);
+  }
+  let limit: number | undefined;
+  if (opts.limit !== undefined) {
+    limit = Number(opts.limit);
+    if (!Number.isInteger(limit) || limit <= 0) {
+      console.error(`--limit must be a positive integer, got: ${opts.limit}`);
+      process.exit(2);
+    }
+  }
+  let perType: number | undefined;
+  if (opts.stratify !== undefined) {
+    perType = Number(opts.stratify);
+    if (!Number.isInteger(perType) || perType <= 0) {
+      console.error(`--stratify must be a positive integer, got: ${opts.stratify}`);
+      process.exit(2);
+    }
+  }
+  const adapterNames = opts.adapters.split(",").map((s) => s.trim()).filter(Boolean);
+  for (const a of adapterNames) {
+    if (!ADAPTERS[a]) {
+      console.error(`unknown adapter: ${a}. options: ${Object.keys(ADAPTERS).join(",")}`);
+      process.exit(2);
+    }
+  }
+  let questions = loadLongMemEval(resolve(opts.data), limit);
+  if (perType) questions = stratifySample(questions, perType);
+  console.log(
+    `loaded ${questions.length} questions, adapters: ${adapterNames.join(",")}, k=${k}`,
+  );
+
+  const outDir = resolve(opts.out);
+  mkdirSync(outDir, { recursive: true });
+  const ndjsonPath = `${outDir}/scores.ndjson`;
+  if (existsSync(ndjsonPath)) writeFileSync(ndjsonPath, "");
+  mkdirSync(dirname(ndjsonPath), { recursive: true });
+
+  const rows: ScoreRow[] = [];
+  for (const adapterName of adapterNames) {
+    const adapter = ADAPTERS[adapterName];
+    console.log(`\n== ${adapter.name} ==`);
+    for (const q of questions) {
+      const t0 = performance.now();
+      const state = await adapter.init(q.haystack);
+      try {
+        const ranked = await adapter.query(q.question, state, k);
+        const latencyMs = performance.now() - t0;
+        const row = scoreQuestion(q, ranked, k, adapter.name, latencyMs);
+        rows.push(row);
+        appendFileSync(ndjsonPath, JSON.stringify(row) + "\n");
+        const mark = row.hit ? "+" : "-";
+        console.log(
+          `  ${mark} ${q.id} [${q.type}] R@${k}=${row.recallAtK.toFixed(2)} (${Math.round(latencyMs)}ms)`,
+        );
+      } finally {
+        if (adapter.teardown) await adapter.teardown(state);
+      }
+    }
+  }
+
+  const agg = aggregate(rows);
+  const summaryPath = `${outDir}/summary.json`;
+  writeFileSync(summaryPath, JSON.stringify(agg, null, 2));
+
+  console.log("\n=== Summary ===");
+  for (const [adapter, stats] of Object.entries(agg.byAdapter)) {
+    console.log(
+      `  ${adapter.padEnd(22)} P@${k}=${stats.p.toFixed(3)} R@${k}=${stats.r.toFixed(3)} hit=${stats.hit}/${stats.n} p50=${Math.round(stats.latencyP50)}ms`,
+    );
+  }
+  console.log(`\nwrote ${ndjsonPath}`);
+  console.log(`wrote ${summaryPath}`);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

eval/runner/score.ts

@@ -0,0 +1,78 @@
+import type { Question, RankedDoc, ScoreRow } from "./types.js";
+
+export function scoreQuestion(
+  q: Question,
+  ranked: RankedDoc[],
+  k: number,
+  adapter: string,
+  latencyMs: number,
+): ScoreRow {
+  const topK = ranked.slice(0, k).map((r) => r.sessionId);
+  const gold = new Set(q.goldSessionIds);
+  const hits = topK.filter((id) => gold.has(id)).length;
+  const precisionAtK = k > 0 ? hits / k : 0;
+  const recallAtK = gold.size === 0 ? 0 : hits / gold.size;
+  const hit = hits > 0;
+  let topGoldRank: number | null = null;
+  for (let i = 0; i < ranked.length; i++) {
+    if (gold.has(ranked[i].sessionId)) {
+      topGoldRank = i + 1;
+      break;
+    }
+  }
+  return {
+    questionId: q.id,
+    questionType: q.type,
+    adapter,
+    k,
+    precisionAtK,
+    recallAtK,
+    hit,
+    topGoldRank,
+    latencyMs,
+  };
+}
+
+export function aggregate(rows: ScoreRow[]): {
+  byAdapter: Record<string, { p: number; r: number; hit: number; n: number; latencyP50: number }>;
+  byType: Record<string, Record<string, { p: number; r: number; hit: number; n: number }>>;
+} {
+  const byAdapter: Record<
+    string,
+    { p: number; r: number; hit: number; n: number; latencyP50: number }
+  > = {};
+  const latencies: Record<string, number[]> = {};
+  for (const r of rows) {
+    const a = (byAdapter[r.adapter] ??= { p: 0, r: 0, hit: 0, n: 0, latencyP50: 0 });
+    a.p += r.precisionAtK;
+    a.r += r.recallAtK;
+    a.hit += r.hit ? 1 : 0;
+    a.n += 1;
+    (latencies[r.adapter] ??= []).push(r.latencyMs);
+  }
+  for (const adapter of Object.keys(byAdapter)) {
+    const a = byAdapter[adapter];
+    a.p = a.p / a.n;
+    a.r = a.r / a.n;
+    const sorted = latencies[adapter].slice().sort((x, y) => x - y);
+    a.latencyP50 = sorted[Math.floor(sorted.length / 2)] ?? 0;
+  }
+  const byType: Record<string, Record<string, { p: number; r: number; hit: number; n: number }>> =
+    {};
+  for (const r of rows) {
+    const t = (byType[r.questionType] ??= {});
+    const a = (t[r.adapter] ??= { p: 0, r: 0, hit: 0, n: 0 });
+    a.p += r.precisionAtK;
+    a.r += r.recallAtK;
+    a.hit += r.hit ? 1 : 0;
+    a.n += 1;
+  }
+  for (const t of Object.keys(byType)) {
+    for (const adapter of Object.keys(byType[t])) {
+      const a = byType[t][adapter];
+      a.p = a.p / a.n;
+      a.r = a.r / a.n;
+    }
+  }
+  return { byAdapter, byType };
+}

eval/runner/types.ts

@@ -0,0 +1,38 @@
+export interface Session {
+  id: string;
+  timestamp?: string;
+  content: string;
+}
+
+export interface Question {
+  id: string;
+  type: string;
+  question: string;
+  answer?: string;
+  goldSessionIds: string[];
+  haystack: Session[];
+}
+
+export interface RankedDoc {
+  sessionId: string;
+  score: number;
+}
+
+export interface Adapter<State = unknown> {
+  name: string;
+  init(sessions: Session[], config?: Record<string, unknown>): Promise<State>;
+  query(q: string, state: State, k: number): Promise<RankedDoc[]>;
+  teardown?(state: State): Promise<void>;
+}
+
+export interface ScoreRow {
+  questionId: string;
+  questionType: string;
+  adapter: string;
+  k: number;
+  precisionAtK: number;
+  recallAtK: number;
+  hit: boolean;
+  topGoldRank: number | null;
+  latencyMs: number;
+}