elizaos--eliza/packages/benchmarks/recall-bench/baseline-1k.json

{
  "benchmark": "recall-bench",
  "generatedAt": "2026-06-29T20:28:52.042Z",
  "corpus": {
    "committed": true,
    "tier": "1k",
    "topics": 40,
    "documents": 1000,
    "queries": 40
  },
  "k": 5,
  "modes": [
    {
      "mode": "document-hybrid",
      "corpusSize": 1000,
      "queries": 40,
      "k": 5,
      "precisionAtK": 0.9500000000000002,
      "recallAtK": 0.9500000000000002,
      "mrr": 0.9875,
      "ndcgAtK": 0.9572638165022521,
      "hitRateAtK": 1,
      "p50LatencyMs": 8.599937000000864,
      "p95LatencyMs": 9.986666999999215,
      "measured": true
    },
    {
      "mode": "document-vector",
      "corpusSize": 1000,
      "queries": 40,
      "k": 5,
      "precisionAtK": 0.9650000000000001,
      "recallAtK": 0.9650000000000001,
      "mrr": 1,
      "ndcgAtK": 0.9738548363300084,
      "hitRateAtK": 1,
      "p50LatencyMs": 5.472680999999284,
      "p95LatencyMs": 6.121880999999121,
      "measured": true
    },
    {
      "mode": "document-keyword",
      "corpusSize": 1000,
      "queries": 40,
      "k": 5,
      "precisionAtK": 0.37,
      "recallAtK": 0.37,
      "mrr": 0.6666666666666667,
      "ndcgAtK": 0.4224540926172685,
      "hitRateAtK": 0.775,
      "p50LatencyMs": 51.79219699999885,
      "p95LatencyMs": 70.65206299999954,
      "measured": true
    },
    {
      "mode": "searchMemories-vector",
      "corpusSize": 1000,
      "queries": 40,
      "k": 5,
      "precisionAtK": 0.9650000000000001,
      "recallAtK": 0.9650000000000001,
      "mrr": 1,
      "ndcgAtK": 0.9738548363300084,
      "hitRateAtK": 1,
      "p50LatencyMs": 5.915468999999575,
      "p95LatencyMs": 6.376758999998856,
      "measured": true
    },
    {
      "mode": "keyword-chat-bm25",
      "corpusSize": 1000,
      "queries": 40,
      "k": 5,
      "precisionAtK": 0.51,
      "recallAtK": 0.51,
      "mrr": 0.7675,
      "ndcgAtK": 0.5472575646212523,
      "hitRateAtK": 0.925,
      "p50LatencyMs": 8.799481000000014,
      "p95LatencyMs": 10.613537999999608,
      "measured": true
    },
    {
      "mode": "facts-provider-keyword",
      "corpusSize": 70,
      "queries": 40,
      "k": 5,
      "precisionAtK": 0.2000000000000001,
      "recallAtK": 1,
      "mrr": 1,
      "ndcgAtK": 1,
      "hitRateAtK": 1,
      "p50LatencyMs": 6.3936720000001515,
      "p95LatencyMs": 7.255925999999818,
      "measured": true
    },
    {
      "mode": "keyword-morph-stemmed",
      "corpusSize": 40,
      "queries": 10,
      "k": 5,
      "precisionAtK": 0.7999999999999999,
      "recallAtK": 1,
      "mrr": 1,
      "ndcgAtK": 1,
      "hitRateAtK": 1,
      "p50LatencyMs": 0.2639870000002702,
      "p95LatencyMs": 0.6226459999998042,
      "measured": true
    },
    {
      "mode": "keyword-morph-unstemmed",
      "corpusSize": 40,
      "queries": 10,
      "k": 5,
      "precisionAtK": 0.04,
      "recallAtK": 0.05,
      "mrr": 0.09166666666666665,
      "ndcgAtK": 0.03633175612671683,
      "hitRateAtK": 0.2,
      "p50LatencyMs": 0.06230900000082329,
      "p95LatencyMs": 0.09761599999910686,
      "measured": true
    },
    {
      "mode": "document-vector-failopen",
      "corpusSize": 1000,
      "queries": 40,
      "k": 5,
      "precisionAtK": 0.37,
      "recallAtK": 0.37,
      "mrr": 0.6666666666666667,
      "ndcgAtK": 0.4224540926172685,
      "hitRateAtK": 0.775,
      "p50LatencyMs": 48.97615299999961,
      "p95LatencyMs": 65.91325499999948,
      "measured": true
    }
  ],
  "failOpen": {
    "vectorRecallAt5": 0.9650000000000001,
    "failOpenRecallAt5": 0.37,
    "recallDrop": 0.5950000000000001,
    "observable": true
  },
  "stemming": {
    "stemmedRecallAt5": 1,
    "unstemmedRecallAt5": 0.05,
    "recallLift": 0.95,
    "observable": true
  },
  "metrics": {
    "overall_recall_at_5": 0.9500000000000002,
    "overall_ndcg_at_5": 0.9572638165022521,
    "overall_p95_latency_ms": 9.986666999999215
  },
  "checks": [
    {
      "name": "document-hybrid.minRecallAt5",
      "value": 0.9500000000000002,
      "budget": 0.85,
      "unit": "ratio",
      "pass": true
    },
    {
      "name": "document-hybrid.minNdcgAt5",
      "value": 0.9572638165022521,
      "budget": 0.85,
      "unit": "ratio",
      "pass": true
    },
    {
      "name": "document-hybrid.maxP95LatencyMs",
      "value": 9.986666999999215,
      "budget": 750,
      "unit": "ms",
      "pass": true
    },
    {
      "name": "document-vector.minRecallAt5",
      "value": 0.9650000000000001,
      "budget": 0.85,
      "unit": "ratio",
      "pass": true
    },
    {
      "name": "document-vector.minNdcgAt5",
      "value": 0.9738548363300084,
      "budget": 0.85,
      "unit": "ratio",
      "pass": true
    },
    {
      "name": "document-vector.maxP95LatencyMs",
      "value": 6.121880999999121,
      "budget": 750,
      "unit": "ms",
      "pass": true
    },
    {
      "name": "document-keyword.minRecallAt5",
      "value": 0.37,
      "budget": 0.25,
      "unit": "ratio",
      "pass": true
    },
    {
      "name": "document-keyword.maxP95LatencyMs",
      "value": 70.65206299999954,
      "budget": 750,
      "unit": "ms",
      "pass": true
    },
    {
      "name": "searchMemories-vector.minRecallAt5",
      "value": 0.9650000000000001,
      "budget": 0.85,
      "unit": "ratio",
      "pass": true
    },
    {
      "name": "searchMemories-vector.minNdcgAt5",
      "value": 0.9738548363300084,
      "budget": 0.85,
      "unit": "ratio",
      "pass": true
    },
    {
      "name": "searchMemories-vector.maxP95LatencyMs",
      "value": 6.376758999998856,
      "budget": 750,
      "unit": "ms",
      "pass": true
    },
    {
      "name": "keyword-chat-bm25.minRecallAt5",
      "value": 0.51,
      "budget": 0.3,
      "unit": "ratio",
      "pass": true
    },
    {
      "name": "keyword-chat-bm25.maxP95LatencyMs",
      "value": 10.613537999999608,
      "budget": 750,
      "unit": "ms",
      "pass": true
    },
    {
      "name": "facts-provider-keyword.minRecallAt5",
      "value": 1,
      "budget": 0.8,
      "unit": "ratio",
      "pass": true
    },
    {
      "name": "facts-provider-keyword.maxP95LatencyMs",
      "value": 7.255925999999818,
      "budget": 750,
      "unit": "ms",
      "pass": true
    },
    {
      "name": "keyword-morph-stemmed.minRecallAt5",
      "value": 1,
      "budget": 0.6,
      "unit": "ratio",
      "pass": true
    },
    {
      "name": "keyword-morph-stemmed.maxP95LatencyMs",
      "value": 0.6226459999998042,
      "budget": 750,
      "unit": "ms",
      "pass": true
    },
    {
      "name": "failOpen.minObservableRecallDrop",
      "value": 0.5950000000000001,
      "budget": 0.3,
      "unit": "ratio",
      "pass": true
    },
    {
      "name": "stemming.minRecallLift",
      "value": 0.95,
      "budget": 0.4,
      "unit": "ratio",
      "pass": true
    }
  ]
}