{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting started with Evals, sampled fixture\n",
    "\n",
    "This fixture mirrors the Cookbook Evals walkthrough but uses tiny local records instead of running a long benchmark.\n"
   ],
   "id": "cell-000"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL = \"gpt-3.5-turbo\"  # stale model kept intentionally for review\n",
    "\n",
    "schema = \"Table cars_data, columns = [Id, MPG, Cylinders, Horsepower, Year]\"\n",
    "examples = [\n",
    "    {\"question\": \"Which cars have more than 100 horsepower?\", \"ideal\": \"SELECT Id FROM cars_data WHERE Horsepower > 100\"},\n",
    "    {\"question\": \"How many cars are from 1970?\", \"ideal\": \"SELECT count(*) FROM cars_data WHERE Year = 1970\"},\n",
    "]\n"
   ],
   "id": "cell-001"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_eval_rows(records):\n",
    "    return [\n",
    "        {\n",
    "            \"input\": [\n",
    "                {\"role\": \"system\", \"content\": f\"Answer with SQLite SQL. {schema}\"},\n",
    "                {\"role\": \"user\", \"content\": record[\"question\"]},\n",
    "            ],\n",
    "            \"ideal\": record[\"ideal\"],\n",
    "        }\n",
    "        for record in records\n",
    "    ]\n",
    "\n",
    "eval_rows = make_eval_rows(examples)\n",
    "eval_rows[0]\n"
   ],
   "id": "cell-002"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Legacy CLI command kept as a repair target; this cell only records it.\n",
    "EVAL_COMMAND = \"oaieval gpt-3.5-turbo spider-sql --max_samples 25\"\n",
    "print(EVAL_COMMAND)\n"
   ],
   "id": "cell-003"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manual log-name placeholder kept as a repair target.\n",
    "log_name = \"240327024443FACXGMKA_gpt-3.5-turbo_spider-sql.jsonl\"  # EDIT THIS\n",
    "local_events = [\n",
    "    {\"type\": \"final_report\", \"data\": {\"accuracy\": 0.5}},\n",
    "    {\"type\": \"sampling\", \"data\": {\"prompt\": eval_rows[0][\"input\"], \"sampled\": \"SELECT Id FROM cars_data WHERE Horsepower > 100\"}},\n",
    "]\n",
    "final_report = next(event[\"data\"] for event in local_events if event[\"type\"] == \"final_report\")\n",
    "final_report\n"
   ],
   "id": "cell-004"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "pygments_lexer": "ipython3"
  },
  "codex_case_study": {
   "source_repo": "https://github.com/openai/openai-cookbook",
   "source_path": "examples/evaluation/Getting_Started_with_OpenAI_Evals.ipynb",
   "source_commit": "96b1d67^",
   "purpose": "Runtime-sampled pre-repair fixture derived from a Cookbook documentation reliability run.",
   "sampling_note": "Compact Evals-style maintenance sample with stale model, deprecated CLI, and result-log issues.",
   "repair_story": {
    "target_iteration": 2,
    "repair_depth": "Two-pass cleanup: first modernize the obvious stale Evals flow, then use validation feedback to remove result-log brittleness.",
    "issue_layers": [
     "stale model",
     "deprecated oaieval command",
     "manual log-name/runtime result dependency"
    ]
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}