wehub-resource-sync
112 lines
3.5 KiB
Plaintext
112 lines
3.5 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Getting started with Evals, sampled fixture\n",
|
|
"\n",
|
|
"This fixture mirrors the Cookbook Evals walkthrough but uses tiny local records instead of running a long benchmark.\n"
|
|
],
|
|
"id": "cell-000"
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"MODEL = \"gpt-3.5-turbo\" # stale model kept intentionally for review\n",
|
|
"\n",
|
|
"schema = \"Table cars_data, columns = [Id, MPG, Cylinders, Horsepower, Year]\"\n",
|
|
"examples = [\n",
|
|
" {\"question\": \"Which cars have more than 100 horsepower?\", \"ideal\": \"SELECT Id FROM cars_data WHERE Horsepower > 100\"},\n",
|
|
" {\"question\": \"How many cars are from 1970?\", \"ideal\": \"SELECT count(*) FROM cars_data WHERE Year = 1970\"},\n",
|
|
"]\n"
|
|
],
|
|
"id": "cell-001"
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def make_eval_rows(records):\n",
|
|
" return [\n",
|
|
" {\n",
|
|
" \"input\": [\n",
|
|
" {\"role\": \"system\", \"content\": f\"Answer with SQLite SQL. {schema}\"},\n",
|
|
" {\"role\": \"user\", \"content\": record[\"question\"]},\n",
|
|
" ],\n",
|
|
" \"ideal\": record[\"ideal\"],\n",
|
|
" }\n",
|
|
" for record in records\n",
|
|
" ]\n",
|
|
"\n",
|
|
"eval_rows = make_eval_rows(examples)\n",
|
|
"eval_rows[0]\n"
|
|
],
|
|
"id": "cell-002"
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Legacy CLI command kept as a repair target; this cell only records it.\n",
|
|
"EVAL_COMMAND = \"oaieval gpt-3.5-turbo spider-sql --max_samples 25\"\n",
|
|
"print(EVAL_COMMAND)\n"
|
|
],
|
|
"id": "cell-003"
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Manual log-name placeholder kept as a repair target.\n",
|
|
"log_name = \"240327024443FACXGMKA_gpt-3.5-turbo_spider-sql.jsonl\" # EDIT THIS\n",
|
|
"local_events = [\n",
|
|
" {\"type\": \"final_report\", \"data\": {\"accuracy\": 0.5}},\n",
|
|
" {\"type\": \"sampling\", \"data\": {\"prompt\": eval_rows[0][\"input\"], \"sampled\": \"SELECT Id FROM cars_data WHERE Horsepower > 100\"}},\n",
|
|
"]\n",
|
|
"final_report = next(event[\"data\"] for event in local_events if event[\"type\"] == \"final_report\")\n",
|
|
"final_report\n"
|
|
],
|
|
"id": "cell-004"
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"name": "python",
|
|
"pygments_lexer": "ipython3"
|
|
},
|
|
"codex_case_study": {
|
|
"source_repo": "https://github.com/openai/openai-cookbook",
|
|
"source_path": "examples/evaluation/Getting_Started_with_OpenAI_Evals.ipynb",
|
|
"source_commit": "96b1d67^",
|
|
"purpose": "Runtime-sampled pre-repair fixture derived from a Cookbook documentation reliability run.",
|
|
"sampling_note": "Compact Evals-style maintenance sample with stale model, deprecated CLI, and result-log issues.",
|
|
"repair_story": {
|
|
"target_iteration": 2,
|
|
"repair_depth": "Two-pass cleanup: first modernize the obvious stale Evals flow, then use validation feedback to remove result-log brittleness.",
|
|
"issue_layers": [
|
|
"stale model",
|
|
"deprecated oaieval command",
|
|
"manual log-name/runtime result dependency"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|