chore: import upstream snapshot with attribution

eval/runner/adapters/agentmemory.ts

@@ -0,0 +1,93 @@
+import type { Adapter, RankedDoc, Session } from "../types.js";
+
+interface AgentMemoryState {
+  baseUrl: string;
+  secret?: string;
+  sessions: Session[];
+  observationToSession: Map<string, string>;
+}
+
+interface RememberResponse {
+  memory?: { id?: string };
+  observationId?: string;
+  id?: string;
+  observation?: { id?: string };
+}
+
+interface SmartSearchResponse {
+  results?: Array<{
+    obsId?: string;
+    id?: string;
+    observationId?: string;
+    sessionId?: string;
+    score?: number;
+    content?: string;
+  }>;
+  observations?: Array<{
+    obsId?: string;
+    id?: string;
+    sessionId?: string;
+    score?: number;
+    content?: string;
+  }>;
+}
+
+function authHeaders(secret?: string): Record<string, string> {
+  const h: Record<string, string> = { "Content-Type": "application/json" };
+  if (secret) h.Authorization = `Bearer ${secret}`;
+  return h;
+}
+
+export const agentmemoryAdapter: Adapter<AgentMemoryState> = {
+  name: "agentmemory-hybrid",
+  async init(sessions, config) {
+    const baseUrl = (config?.baseUrl as string) ?? process.env.AGENTMEMORY_BASE_URL ?? "http://localhost:3111";
+    const secret = (config?.secret as string) ?? process.env.AGENTMEMORY_SECRET;
+    const observationToSession = new Map<string, string>();
+    for (const s of sessions) {
+      const res = await fetch(`${baseUrl}/agentmemory/remember`, {
+        method: "POST",
+        headers: authHeaders(secret),
+        body: JSON.stringify({
+          content: s.content,
+          type: "eval-session",
+          concepts: [s.id],
+        }),
+      });
+      if (!res.ok) {
+        throw new Error(`remember failed for ${s.id}: ${res.status} ${await res.text()}`);
+      }
+      const body = (await res.json()) as RememberResponse;
+      const obsId =
+        body.memory?.id ?? body.observationId ?? body.id ?? body.observation?.id;
+      if (obsId) observationToSession.set(obsId, s.id);
+    }
+    return { baseUrl, secret, sessions, observationToSession };
+  },
+  async query(q, state, k) {
+    const res = await fetch(`${state.baseUrl}/agentmemory/smart-search`, {
+      method: "POST",
+      headers: authHeaders(state.secret),
+      body: JSON.stringify({ query: q, limit: Math.max(k * 10, 50) }),
+    });
+    if (!res.ok) {
+      throw new Error(`smart-search failed: ${res.status} ${await res.text()}`);
+    }
+    const body = (await res.json()) as SmartSearchResponse;
+    const rows = body.results ?? body.observations ?? [];
+    const ranked: RankedDoc[] = [];
+    const seen = new Set<string>();
+    for (const row of rows) {
+      let sessionId = row.sessionId;
+      if (!sessionId) {
+        const memId = row.obsId ?? row.id ?? row.observationId;
+        sessionId = memId ? state.observationToSession.get(memId) : undefined;
+      }
+      if (!sessionId || seen.has(sessionId)) continue;
+      seen.add(sessionId);
+      ranked.push({ sessionId, score: row.score ?? 0 });
+      if (ranked.length >= k) break;
+    }
+    return ranked;
+  },
+};

eval/runner/adapters/grep.ts

@@ -0,0 +1,36 @@
+import type { Adapter, RankedDoc, Session } from "../types.js";
+
+interface GrepState {
+  sessions: Session[];
+}
+
+function tokenize(s: string): string[] {
+  return s
+    .toLowerCase()
+    .replace(/[^a-z0-9_]+/g, " ")
+    .split(/\s+/)
+    .filter((t) => t.length > 2);
+}
+
+export const grepAdapter: Adapter<GrepState> = {
+  name: "grep",
+  async init(sessions) {
+    return { sessions };
+  },
+  async query(q, state, k) {
+    const terms = tokenize(q);
+    const scored: RankedDoc[] = [];
+    for (const s of state.sessions) {
+      const body = s.content.toLowerCase();
+      let hits = 0;
+      for (const t of terms) {
+        if (body.includes(t)) hits += 1;
+      }
+      if (hits > 0) {
+        scored.push({ sessionId: s.id, score: hits });
+      }
+    }
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, k);
+  },
+};

eval/runner/adapters/vector.ts

@@ -0,0 +1,108 @@
+import type { Adapter, RankedDoc, Session } from "../types.js";
+
+interface VectorState {
+  sessions: Session[];
+  embeddings: Float32Array[];
+}
+
+const OPENAI_URL = "https://api.openai.com/v1/embeddings";
+const MODEL = "text-embedding-3-small";
+const DIM = 1536;
+
+async function embed(text: string, apiKey: string): Promise<Float32Array> {
+  const res = await fetch(OPENAI_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({ input: text, model: MODEL }),
+  });
+  if (!res.ok) {
+    throw new Error(`OpenAI embed failed: ${res.status} ${await res.text()}`);
+  }
+  const data = (await res.json()) as { data: Array<{ embedding: number[] }> };
+  return Float32Array.from(data.data[0].embedding);
+}
+
+async function embedBatch(texts: string[], apiKey: string): Promise<Float32Array[]> {
+  const res = await fetch(OPENAI_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({ input: texts, model: MODEL }),
+  });
+  if (!res.ok) {
+    throw new Error(`OpenAI batch embed failed: ${res.status} ${await res.text()}`);
+  }
+  const data = (await res.json()) as { data: Array<{ embedding: number[]; index: number }> };
+  if (!Array.isArray(data.data) || data.data.length !== texts.length) {
+    throw new Error(
+      `OpenAI batch embed: expected ${texts.length} embeddings, got ${data.data?.length ?? 0}`,
+    );
+  }
+  const out = new Array<Float32Array>(texts.length);
+  for (const row of data.data) {
+    if (
+      !Number.isInteger(row.index) ||
+      row.index < 0 ||
+      row.index >= texts.length ||
+      out[row.index] !== undefined
+    ) {
+      throw new Error(`OpenAI batch embed: invalid or duplicate index ${row.index}`);
+    }
+    if (!Array.isArray(row.embedding) || row.embedding.length === 0) {
+      throw new Error(`OpenAI batch embed: empty embedding at index ${row.index}`);
+    }
+    out[row.index] = Float32Array.from(row.embedding);
+  }
+  return out;
+}
+
+function cosine(a: Float32Array, b: Float32Array): number {
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+export const vectorAdapter: Adapter<VectorState> = {
+  name: "vector",
+  async init(sessions) {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) throw new Error("OPENAI_API_KEY required for vector adapter");
+    const embeddings: Float32Array[] = new Array(sessions.length);
+    const BATCH = 50;
+    for (let i = 0; i < sessions.length; i += BATCH) {
+      const batch = sessions.slice(i, i + BATCH);
+      const vecs = await embedBatch(
+        batch.map((s) => s.content.slice(0, 8000)),
+        apiKey,
+      );
+      for (let j = 0; j < vecs.length; j++) embeddings[i + j] = vecs[j];
+    }
+    if (embeddings.length > 0 && embeddings[0].length !== DIM) {
+      throw new Error(`unexpected embedding dim: ${embeddings[0].length}`);
+    }
+    return { sessions, embeddings };
+  },
+  async query(q, state, k) {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) throw new Error("OPENAI_API_KEY required for vector adapter");
+    const qvec = await embed(q, apiKey);
+    const scored: RankedDoc[] = state.sessions.map((s, i) => ({
+      sessionId: s.id,
+      score: cosine(qvec, state.embeddings[i]),
+    }));
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, k);
+  },
+};