comet-ml--opik/apps/opik-documentation/documentation/docs/cookbook/optimizer_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NTaMBjChz7m5"
   },
   "source": [
    "# Introduction to Opik Agent Optimizer\n",
    "\n",
    "You will need:\n",
    "\n",
    "1. A Comet account, for seeing Opik visualizations (free!) - [comet.com](https://comet.com)\n",
    "2. An OpenAI account, for using an LLM\n",
    "[platform.openai.com/settings/organization/api-keys](https://platform.openai.com/settings/organization/api-keys)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yM1lU0dBBnJs"
   },
   "source": [
    "## Setup\n",
    "\n",
    "This pip-install takes about a minute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2Tx6HwuU1rB4",
    "outputId": "0ccc9edb-9a9b-4e91-8937-7894e16aaf67"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.2/41.2 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m153.1/153.1 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.1/126.1 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m136.0/136.0 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m681.5/681.5 kB\u001b[0m \u001b[31m22.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.9/8.9 MB\u001b[0m \u001b[31m87.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m400.9/400.9 kB\u001b[0m \u001b[31m23.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m247.0/247.0 kB\u001b[0m \u001b[31m14.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m70.0/70.0 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m786.8/786.8 kB\u001b[0m \u001b[31m31.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m79.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.1/66.1 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h"
     ]
    }
   ],
   "source": [
    "%pip install opik-optimizer --quiet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SHPEMgER-wEk"
   },
   "source": [
    "Let's make sure we have the correct version:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "id": "pVgkLU3v2KD8",
    "outputId": "5c9b9f48-ecff-4665-f6ca-2c291896ec37"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'1.0.6'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import opik_optimizer\n",
    "\n",
    "opik_optimizer.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "92BKIsT0-wEl"
   },
   "source": [
    "The version should be 1.0.6 or greater.\n",
    "\n",
    "[Comet](https://www.comet.com/site?from=llm&utm_source=opik) provides a hosted version of the Opik platform, [simply create an account](https://www.comet.com/signup?from=llm&utm_source=opik&utm_medium=colab&utm_content=langchain&utm_campaign=opik) and grab your API Key.\n",
    "\n",
    "> You can also run the Opik platform locally, see the [installation guide](https://www.comet.com/docs/opik/self-host/overview/?from=llm&utm_source=opik) for more information.\n",
    "\n",
    "Enter your Comet API key, followed by \"Y\" to use your own workspace:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "H0DNm-un_0Np",
    "outputId": "de3bdcf7-1c0d-4d30-b0da-9accb26d2a99"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "OPIK: Your Opik API key is available in your account settings, can be found at https://www.comet.com/api/my/settings/ for Opik cloud\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please enter your Opik API key:··········\n",
      "Do you want to use \"doug-comet\" workspace? (Y/n)y\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "OPIK: Configuration saved to file: /root/.opik.config\n"
     ]
    }
   ],
   "source": [
    "import opik\n",
    "\n",
    "# Configure Opik\n",
    "opik.configure()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wjUurWGr-wEl"
   },
   "source": [
    "For this example, we'll use OpenAI models, so we need to set our OpenAI API key:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vN72mHQy_7Ou",
    "outputId": "10132aae-e3bd-4b2f-c27f-cf0bcb4c2ecd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter your OpenAI API key: ··········\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import getpass\n",
    "\n",
    "if \"OPENAI_API_KEY\" not in os.environ:\n",
    "    os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter your OpenAI API key: \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "n1r1oI-z-wEl"
   },
   "source": [
    "To optimize any prompt, we'll need three basic things:\n",
    "\n",
    "1. A starting prompt\n",
    "2. A metric\n",
    "3. A dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zVK_KLEXbyqu"
   },
   "source": [
    "## The Dataset\n",
    "\n",
    "In this experiment, we are going to use the **HotPotQA** dataset. This dataset was designed to be difficult for regular LLMs to handle. This dataset is called a \"**multi-hop**\" dataset because answering the questions involves multiple reasoning steps and multiple tool calls, where the LLM needs to infer relationships, combine information, or draw conclusions based on the combined context.\n",
    "\n",
    "Example:\n",
    "\n",
    "> \"What are the capitals of the states that border California?\"\n",
    "\n",
    "You'd need to find which states border California, and then lookup each state's capital.\n",
    "\n",
    "The dataset has about 113,000 such crowd-sourced questions that are constructed to require the introductory paragraphs of two Wikipedia articles to answer.\n",
    "\n",
    "**NOTE:** The name \"HotPot\" comes from the restaurant where the authors came up with the idea of the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4gCj_2M3A37D",
    "outputId": "5ae09581-fc5f-4e44-9821-0154eddf6238"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipython-input-70379200.py:3: DeprecationWarning: This function is deprecated. Please use the datasets directly from opik_optimizer.datasets module instead. For example: opik_optimizer.datasets.truthful_qa() or opik_optimizer.datasets.rag_hallucination()\n",
      "  dataset = get_or_create_dataset(\"hotpot-300\")\n"
     ]
    }
   ],
   "source": [
    "from opik_optimizer.demo import get_or_create_dataset\n",
    "\n",
    "dataset = get_or_create_dataset(\"hotpot-300\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YvqslJnA9RMQ"
   },
   "source": [
    "Let's take a look at some dataset items:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KGZoscWiBInJ",
    "outputId": "2bacd378-33b2-41f8-e83c-4dc01d113ef5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '0198ccfe-efba-7d28-acad-727ea577d76c',\n",
       " 'question': 'Are Smyrnium and Nymania both types of plant?',\n",
       " 'answer': 'yes'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rows = dataset.get_items()\n",
    "rows[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "c6Xh04Cv-wEm"
   },
   "source": [
    "We see that each item has a \"question\" and \"answer\". Some of the answers are short and direct, and some of them are more complicated:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8MQC4TllBKgT",
    "outputId": "c47244e8-fa89-4ea5-87fd-4c39b93d9674"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '0198ccfe-efb9-70ff-a616-61889599f2e4',\n",
       " 'question': 'That Darn Cat! and Never a Dull Moment were both produced by what studio?',\n",
       " 'answer': 'Walt Disney Productions'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rows[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "feVF8adydeyH"
   },
   "source": [
    "## Opik Project\n",
    "\n",
    "All LLM traces in Opik are saved in a \"project\". We'll put them all in the following project name:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "gTTZ5apsPgUg"
   },
   "outputs": [],
   "source": [
    "project_name = \"optimize-workshop-2025\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VEEHnyrcdjVw"
   },
   "source": [
    "## The Metric\n",
    "\n",
    "Choosing a good metric for optimization is tricky. For these examples, we'll pick one that will allow us to show improvement, and also provide a gradient of scores. In general though, this metric isn't the best for optimization runs.\n",
    "\n",
    "We'll use \"Edit Distance\" AKA \"Levenshtein Distance\":"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "T3BKOXxPOpkI"
   },
   "outputs": [],
   "source": [
    "from opik.evaluation.metrics import LevenshteinRatio\n",
    "\n",
    "metric = LevenshteinRatio(project_name=project_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BF9czV9Ue02R"
   },
   "source": [
    "The metric takes two things: the `output` of the LLM and the `reference` (the truth):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_hMQuJsSexkD",
    "outputId": "8ab33c1f-4f15-4054-bfd2-67a21c7fc0c7"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "OPIK: Started logging traces to the \"optimize-workshop-2025\" project at https://www.comet.com/opik/api/v1/session/redirect/projects/?trace_id=0198cd48-3025-7dd2-a0af-2ac01850a7b3&path=aHR0cHM6Ly93d3cuY29tZXQuY29tL29waWsvYXBpLw==.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ScoreResult(name='levenshtein_ratio_metric', value=1.0, reason=None, metadata=None, scoring_failed=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric.score(\"Hello\", \"Hello\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hNleaL5ZPLaA",
    "outputId": "959c59fb-b0d5-4aa6-906d-485bccdfb94f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ScoreResult(name='levenshtein_ratio_metric', value=0.9090909090909091, reason=None, metadata=None, scoring_failed=False)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric.score(\"Hello!\", \"Hello\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2eI1S-Q_-SwO"
   },
   "source": [
    "The edit distance between \"Hello!\" and \"Hello\" is 1. Here is how the .91 is computed:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7YEltgab4jPs",
    "outputId": "0bdab646-5181-473f-b797-3137ff435fb2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9090909090909091"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edit_distance = 1\n",
    "\n",
    "1 - edit_distance / (len(\"Hello1\") + len(\"Hello\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p3gIizQt47AP"
   },
   "source": [
    "For more information see: [Levenshtein Distance](https://en.wikipedia.org/wiki/Levenshtein_distance)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JarFSV9jb7jD"
   },
   "source": [
    "## Configuation\n",
    "\n",
    "To create the necesary configurations for using an Opik Agent Optimizer, you'll need three things:\n",
    "\n",
    "1. An initial prompt\n",
    "2. A metric wrapper\n",
    "\n",
    "We're going to start with a pretty bad prompt... so we can optimize it!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "xxDUfpvgPct7"
   },
   "outputs": [],
   "source": [
    "from opik_optimizer import ChatPrompt\n",
    "\n",
    "initial_prompt = ChatPrompt(\n",
    "    system=\"Provide an answer to the question\",\n",
    "    user=\"{question}\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Oia79J_e-vSY"
   },
   "source": [
    "The metric wrapper:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "ET_umnWdPmjm"
   },
   "outputs": [],
   "source": [
    "def levenshtein_ratio(dataset_item, llm_output):\n",
    "    metric = LevenshteinRatio(project_name=project_name)\n",
    "    return metric.score(\n",
    "        reference=dataset_item[\"answer\"], # This must match dataset field\n",
    "        output=llm_output,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2aDp9vjw5hIQ"
   },
   "source": [
    "As you can see the metric wrapper is composed of our chosen metric. It takes a dataset item (a dictionary row from the dataset) and the output from the LLM. You should replace \"answer\" when using a different dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VBNAMIglcVIo"
   },
   "source": [
    "## FewShotBayesianOptimizer\n",
    "\n",
    "The FewShotBayesianOptimizer name indicates two things:\n",
    "\n",
    "1. It will produce Chat Prompts, or FewShot examples as described in the slides.\n",
    "2. Secondly, it describes how it searches for the best set of these FewShot examples.\n",
    "\n",
    "To use this optimizer, we import it and create an instance, passing in the project name and model parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "EbaysjG1PRNd"
   },
   "outputs": [],
   "source": [
    "from opik_optimizer import (\n",
    "    FewShotBayesianOptimizer,\n",
    ")\n",
    "\n",
    "optimizer = FewShotBayesianOptimizer(\n",
    "    model=\"openai/gpt-4o\",  # LiteLLM name\n",
    "    min_examples=3,\n",
    "    max_examples=8,\n",
    "    n_threads=4,\n",
    "    seed=42,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UtFggdB67tmB"
   },
   "source": [
    "Ok, let's optimize that prompt!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "89b216619d2b44e98f01786b4d0c7b85",
      "346d3556af2441568682425b3cd6dc7e",
      "0e569d9a712c4f07a2663acbeabf9839",
      "9dee9555a1104b5fa7f776abc8d4bbce",
      "ba3efe55644442c1bc9c00470ef281cc",
      "74ad014e881645f59dfa7cd0dfc16cce",
      "a1f8c6c1822f4251a3c52b64a5f4ccb8",
      "0a3d0d3a7def4ecea085cf7f662b1ab0",
      "44d08cd6b96e4123af6009ea21aff437",
      "d4df8250b36a4a69bc11e0981580af2c",
      "64ce01dfd3a84acc89b6ad73dcbb5992",
      "8c9f8ee7da4b4231a85a84b5e7b288cd",
      "21871aba89694be4966217f8b2be767b",
      "cd9d5f524695486597117e3e1c032477",
      "5ab5f2b81f5b49789f10921d1a8db1b2",
      "a7834c88f90048f585b26be280a5ad85",
      "bb45492eb3f44719b71eab1470491c0a",
      "ec2a38c2968c4a26b44b15f66727e2ae",
      "883b9f63efd34b53b4a36fa298a05c9c",
      "e06a3e0110cf40d19bb92ef795c9cadb",
      "d630203df2634807a0ea69f9148ce8c1",
      "f8217d6f0a3c462b855271d9f6c1a8a7"
     ]
    },
    "id": "3Q3ENgSARX2m",
    "outputId": "81d3ba80-f4fa-44a5-d33d-ccb2ea40a7ea"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "╭────────────────────────────────────────────────────────────────────╮\n",
      "│ \u001b[32m● \u001b[0mRunning Opik Evaluation - \u001b[34mFewShotBayesianOptimizer\u001b[0m               │\n",
      "│                                                                    │\n",
      "│ -> View optimization details \u001b]8;id=855572;https://www.comet.com/opik/api/v1/session/redirect/optimizations/?optimization_id=0198cd53-9f2d-7650-910e-d0bc21b526dd&dataset_id=0198ccfe-ec43-7649-97ad-fe00f0d2d788&path=aHR0cHM6Ly93d3cuY29tZXQuY29tL29waWsvYXBpLw==\u001b\\in your Opik dashboard\u001b]8;;\u001b\\                │\n",
      "╰────────────────────────────────────────────────────────────────────╯\n",
      "\n",
      "\n",
      "> Let's optimize the prompt:\n",
      "\n",
      "\u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "\u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "\u001b[2m│\u001b[0m  Provide an answer to the question                                 \u001b[2m│\u001b[0m\n",
      "\u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "\u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "\u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "\u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "\u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "\u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "\u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "\n",
      "Using FewShotBayesianOptimizer with the parameters: \n",
      "\u001b[2m  - metric: \u001b[0m\u001b[36mlevenshtein_ratio\u001b[0m\n",
      "\u001b[2m  - n_trials: \u001b[0m\u001b[36m10\u001b[0m\n",
      "\u001b[2m  - n_samples: \u001b[0m\u001b[36m50\u001b[0m\n",
      "\n",
      "\n",
      "> First we will establish the baseline performance:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89b216619d2b44e98f01786b4d0c7b85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m  Baseline score was: 0.1529.\u001b[0m\n",
      "\n",
      "> Let's add a placeholder for few-shot examples in the messages:\n",
      "\u001b[2;33m│    Created the prompt template:\u001b[0m\n",
      "\u001b[2;33m│\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Provide an answer to the question.                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ## Examples                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  FEW_SHOT_EXAMPLE_PLACEHOLDER                                      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│\n",
      "│   With the FEW_SHOT_EXAMPLE_PLACEHOLDER following the format:\n",
      "│    \u001b\u001b[1m[\u001b[0m2m╭────────────────────────────────────────────────────────────────────╮\u001b\u001b[1m[\u001b[0m0m\n",
      "│    \u001b\u001b[1m[\u001b[0m2m│\u001b\u001b[1m[\u001b[0m0m ### Example                                                        \u001b\u001b[1m[\u001b[0m2m│\u001b\u001b[1m[\u001b[0m0m\n",
      "│    \u001b\u001b[1m[\u001b[0m2m│\u001b\u001b[1m[\u001b[0m0m                                                                    \u001b\u001b[1m[\u001b[0m2m│\u001b\u001b[1m[\u001b[0m0m\n",
      "│    \u001b\u001b[1m[\u001b[0m2m│\u001b\u001b[1m[\u001b[0m0m Question: \u001b[1m{\u001b[0mquestion\u001b[1m}\u001b[0m                                               \u001b\u001b[1m[\u001b[0m2m│\u001b\u001b[1m[\u001b[0m0m\n",
      "│    \u001b\u001b[1m[\u001b[0m2m│\u001b\u001b[1m[\u001b[0m0m Answer: \u001b[1m{\u001b[0manswer\u001b[1m}\u001b[0m                                                   \u001b\u001b[1m[\u001b[0m2m│\u001b\u001b[1m[\u001b[0m0m\n",
      "│    \u001b\u001b[1m[\u001b[0m2m│\u001b\u001b[1m[\u001b[0m0m                                                                    \u001b\u001b[1m[\u001b[0m2m│\u001b\u001b[1m[\u001b[0m0m\n",
      "│    \u001b\u001b[1m[\u001b[0m2m╰────────────────────────────────────────────────────────────────────╯\u001b\u001b[1m[\u001b[0m0m\n",
      "\n",
      "\n",
      "> Starting the optimization run\n",
      "│\n",
      "│ - Starting optimization round 1 of 10\n",
      "│\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Provide an answer to the question.                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ## Examples                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What novel did an author who was featured in the        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Voices of Ghana and later became a journalist write?              \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: The Gab Boys                                              \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What were the dates of the campaign that eventually     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  led to the Treaty of Roskilde?                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: between 30 January and 8 February 1658                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Are Wanding Town and Jiujiang, Guangdong both frontier  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  towns?                                                            \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: no                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: The Knicks–Nuggets was the most penalized on-court      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  fight in the NBA since an altercation that occured in what city?  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Auburn Hills                                              \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What year was the man whose car failed to arrive to     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  the 1950 Indianapolis 500 first the Italian champion?             \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: 1937                                                      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e569d9a712c4f07a2663acbeabf9839",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m│    Trial 1 - score was: 0.3231 (111.27%).\u001b[0m\n",
      "\u001b[32m│\u001b[0m\n",
      "│ - Starting optimization round 2 of 10\n",
      "│\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Provide an answer to the question.                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ## Examples                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Ramfis Trujillo was a close friend of the racecar       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  driver and diplomat of what nationality?                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Dominican                                                 \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Which airport Rapid City Regional Airport and           \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Chattanooga Metropolitan Airport is closer to its local city ?    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Chattanooga Metropolitan Airport                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Are Hüsker Dü and The Colourist both musical bands?     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: yes                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Which band from East London had Steve Marriott as a     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  band member?                                                      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Small Faces                                               \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Hae Min Lee's body was found in a park that covers how  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  many acres ?                                                      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: 1216 acre                                                 \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba3efe55644442c1bc9c00470ef281cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m│    Trial 2 - score was: 0.2653 (73.46%).\u001b[0m\n",
      "\u001b[32m│\u001b[0m\n",
      "│ - Starting optimization round 3 of 10\n",
      "│\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Provide an answer to the question.                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ## Examples                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: The 6th Canadian Infantry Division was originally       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  scheduled to participate in this proposed Allied plan for the     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  invasion of Japan near the end of World War II but the plan was   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  abandoned after the Soviet invasion of which region?              \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Manchuria                                                 \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: This group encompassed a federation of Alpine tribes,   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  including the Calucones, for about 6 centuries starting around    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  500 BC.                                                           \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: The Raeti                                                 \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Seiko Holdings Corporation was formerly known as which  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  watch and jewlery shop?                                           \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: K. Hattori                                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: The final episode in \"A Song of Ass and Fire,\" season   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  was what number episode?                                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: ninth                                                     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1f8c6c1822f4251a3c52b64a5f4ccb8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m│    Trial 3 - score was: 0.2539 (66.02%).\u001b[0m\n",
      "\u001b[32m│\u001b[0m\n",
      "│ - Starting optimization round 4 of 10\n",
      "│\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Provide an answer to the question.                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ## Examples                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What Nigerian president appointed the Managing          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Director of the NPA in 2016?                                      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Muhammadu Buhari                                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What sport does Brett Bech NOR Las Vegas Outlaws have   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  in common?                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: football                                                  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Seiko Holdings Corporation was formerly known as which  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  watch and jewlery shop?                                           \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: K. Hattori                                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Were the films Meet the Robinsons and Blackbeard's      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Ghost both based on novels?                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: yes                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: In which competition held in Buenos Aires, Argentina    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  did Stanly Stanczyk win a gold medal?                             \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: The 1951 Pan American Games                               \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Which opera was first performed, Don Quichotte or       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Euridice?                                                         \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Euridice                                                  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What actress from the 1999 movie The Virgin Suicides    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  has won two golden globes and been nominated for an academy       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  award?                                                            \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Mary Kathleen Turner                                      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Laura S. Walker State Park is located near a swamp      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  that has a size of how many acres ?                               \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: 438,000 acre                                              \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44d08cd6b96e4123af6009ea21aff437",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m│    Trial 4 - score was: 0.3326 (117.47%).\u001b[0m\n",
      "\u001b[32m│\u001b[0m\n",
      "│ - Starting optimization round 5 of 10\n",
      "│\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Provide an answer to the question.                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ## Examples                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: The Knicks–Nuggets was the most penalized on-court      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  fight in the NBA since an altercation that occured in what city?  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Auburn Hills                                              \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Which singer, born in 1977, shared the stage with an    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  American bluegrass singer, songwriter, and multi-instrumentalist  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  known as the \"the new Queen of Bluegrass\"?                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Rebecca Rippy                                             \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: WRBJ-FM is licensed to which Rankin County,             \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Mississippi town?                                                 \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Brandon                                                   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Ryan Blair plays at midfield for which Welsh football   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  club?                                                             \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Swansea City                                              \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What movie did \"the king of cool\" play in with Bud      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Ekins as his stunt double?                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: \"The Great Escape\"                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Rectrix Aviation's jet charter business involves what   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  service?                                                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: renting an entire aircraft                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64ce01dfd3a84acc89b6ad73dcbb5992",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m│    Trial 5 - score was: 0.3090 (102.07%).\u001b[0m\n",
      "\u001b[32m│\u001b[0m\n",
      "│ - Starting optimization round 6 of 10\n",
      "│\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Provide an answer to the question.                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ## Examples                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: In which region did the settlers have conflict with     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  the Mexican government where it escalated to a rebellion led by   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  John Dunn Hunter?                                                 \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Texas                                                     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Reggie Grimes played for New England Patriots in 2000,  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  and where did he play college football?                           \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Alabama                                                   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Northrop F-5F aggressor squadron is based in an air     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  station that is designated as a superfund site due to what ?      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: a number of soil and groundwater contaminants, including  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  asbestos                                                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: So Long, Scarecrow is titled in reference to which      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  1939 musical fantasy film?                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: The Wizard of Oz                                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21871aba89694be4966217f8b2be767b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m│    Trial 6 - score was: 0.3309 (116.37%).\u001b[0m\n",
      "\u001b[32m│\u001b[0m\n",
      "│ - Starting optimization round 7 of 10\n",
      "│\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Provide an answer to the question.                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ## Examples                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: One Night in Istanbul stars a British actor and         \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  recording artist who is best known for his portrayal of Jonah     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Kirby on what BBC One drama?                                      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Waterloo Road                                             \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Alicia Gräfin is best known for her role in a war film  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  directed by who?                                                  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: David Ayer                                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What comedy film from 1994 starred actress Heather      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Graham alongside James Le Gros?                                   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Don't Do It                                               \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Are Hüsker Dü and The Colourist both musical bands?     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: yes                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Who was the son of British radio and television         \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  presenter and historian best known as an analyst of election      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  results?                                                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Dan Snow                                                  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Rectrix Aviation's jet charter business involves what   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  service?                                                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: renting an entire aircraft                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What U.S. court presided over both the Slaughter-House  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Cases and Reed v. Reed?                                           \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: the Supreme Court                                         \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ab5f2b81f5b49789f10921d1a8db1b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m│    Trial 7 - score was: 0.4317 (182.34%).\u001b[0m\n",
      "\u001b[32m│\u001b[0m\n",
      "│ - Starting optimization round 8 of 10\n",
      "│\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Provide an answer to the question.                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ## Examples                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: \"He Says the Same Things to Me\" is a song by a country  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  music singer born in what year?                                   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: 1931                                                      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Are Norma and Idomeneo both operas by Bellini?          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: no                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What comedy film from 1994 starred actress Heather      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Graham alongside James Le Gros?                                   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Don't Do It                                               \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What role does Ghanashyam Nayak play in India's         \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  longest running sitcom serial?                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Natwarlal Prabhashankar Undhaiwala                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Which professional basketball player won a NBA          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  championship and spoke positively on the book \"the 21             \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  irrefutable laws of leadership\"?                                  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Harrison Barnes                                           \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Which science fiction trilogy has in the lead role an   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  actress who had a voice role in the 3D film Peter Rabbit?         \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Star Wars                                                 \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What's the present day English county of                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Skelton-in-Cleveland, recorded in the Domesday Book and site of   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  a 12th-century castle?                                            \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: North Yorkshire                                           \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Who was the son of British radio and television         \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  presenter and historian best known as an analyst of election      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  results?                                                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Dan Snow                                                  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb45492eb3f44719b71eab1470491c0a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m│    Trial 8 - score was: 0.3991 (160.97%).\u001b[0m\n",
      "\u001b[32m│\u001b[0m\n",
      "│ - Starting optimization round 9 of 10\n",
      "│\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Provide an answer to the question.                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ## Examples                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What American singer born 1951 does Jim Rooney  have    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  credits for?                                                      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Iris Luella DeMent                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What was the gang merged into by the gang leader born   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  on November 30, 1950?                                             \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Black Gangster Disciple Nation (BGDN)                     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Which television series did the writer of \"Business     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  School\" write for first, The Office or The Simpsons?              \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: The Simpsons                                              \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Widows' Peak stars what English actress commonly known  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  as  Dame                                                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Joan Plowright                                            \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: That Darn Cat! and Never a Dull Moment were both        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  produced by what studio?                                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Walt Disney Productions                                   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Who did Holly Dunn record \"Daddy's Hands\" for?          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: MTM Records                                               \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: How many novels has the author of the novel Child of    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  God written?                                                      \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: He has written ten novels                                 \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "883b9f63efd34b53b4a36fa298a05c9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m│    Trial 9 - score was: 0.2753 (80.06%).\u001b[0m\n",
      "\u001b[32m│\u001b[0m\n",
      "│ - Starting optimization round 10 of 10\n",
      "│\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m──────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Provide an answer to the question.                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ## Examples                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Who is the wife of Charlemagne who is a step mother to  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Pepin the Hunchback?                                              \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Hildegard                                                 \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Yazdegerd II fought wars against which Central Asian    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  people based in Bactria?                                          \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Hephthalite Empire                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: Who is the wife of Charlemagne who is a step mother to  \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Pepin the Hunchback?                                              \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Hildegard                                                 \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What sport does 1989–90 Sacramento Kings season and     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Rodney McCray have in common?                                     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: basketball                                                \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  ### Example                                                       \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Question: What was the gang merged into by the gang leader born   \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  on November 30, 1950?                                             \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  Answer: Black Gangster Disciple Nation (BGDN)                     \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│    \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m────────────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m  {question}                                                        \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m│\u001b[0m                                                                    \u001b[2m│\u001b[0m\n",
      "│    \u001b[2m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "│\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d630203df2634807a0ea69f9148ce8c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m│    Trial 10 - score was: 0.3185 (108.27%).\u001b[0m\n",
      "\u001b[32m│\u001b[0m\n",
      "\n",
      "> Optimization complete\n",
      "\n",
      "\u001b[32m╭─\u001b[0m\u001b[32m Optimization results \u001b[0m\u001b[32m────────────────────────────────────────────\u001b[0m\u001b[32m─╮\u001b[0m\n",
      "\u001b[32m│\u001b[0m                                                                    \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[1;32mPrompt was optimized and improved from 0.1529 to 0.4317 \u001b[0m          \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[1;32m(182.34%)\u001b[0m                                                         \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m                                                                    \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  Optimized prompt:                                                 \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m╭─\u001b[0m\u001b[2m system \u001b[0m\u001b[2m────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Provide an answer to the question.                          \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  ## Examples                                                 \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  ### Example                                                 \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Question: One Night in Istanbul stars a British actor and   \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  recording artist who is best known for his portrayal of     \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Jonah Kirby on what BBC One drama?                          \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Answer: Waterloo Road                                       \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  ### Example                                                 \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Question: Alicia Gräfin is best known for her role in a     \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  war film directed by who?                                   \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Answer: David Ayer                                          \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  ### Example                                                 \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Question: What comedy film from 1994 starred actress        \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Heather Graham alongside James Le Gros?                     \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Answer: Don't Do It                                         \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  ### Example                                                 \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Question: Are Hüsker Dü and The Colourist both musical      \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  bands?                                                      \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Answer: yes                                                 \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  ### Example                                                 \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Question: Who was the son of British radio and television   \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  presenter and historian best known as an analyst of         \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  election results?                                           \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Answer: Dan Snow                                            \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  ### Example                                                 \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Question: Rectrix Aviation's jet charter business involves  \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  what service?                                               \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Answer: renting an entire aircraft                          \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  ### Example                                                 \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Question: What U.S. court presided over both the            \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Slaughter-House Cases and Reed v. Reed?                     \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  Answer: the Supreme Court                                   \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m╰──────────────────────────────────────────────────────────────╯\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m╭─\u001b[0m\u001b[2m user \u001b[0m\u001b[2m──────────────────────────────────────────────────────\u001b[0m\u001b[2m─╮\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m  {question}                                                  \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m│\u001b[0m                                                              \u001b[2m│\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m  \u001b[2m╰──────────────────────────────────────────────────────────────╯\u001b[0m  \u001b[32m│\u001b[0m\n",
      "\u001b[32m│\u001b[0m                                                                    \u001b[32m│\u001b[0m\n",
      "\u001b[32m╰────────────────────────────────────────────────────────────────────╯\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "result = optimizer.optimize_prompt(\n",
    "    prompt=initial_prompt,\n",
    "    dataset=dataset,\n",
    "    metric=levenshtein_ratio,\n",
    "    n_trials=10,\n",
    "    n_samples=50,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "nJVmhi5Gcnkj",
    "outputId": "db4123d4-578f-48c4-ebc3-8486baa1dfb8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33m╔═\u001b[0m\u001b[33m════════════════════════════════════════════\u001b[0m\u001b[33m \u001b[0m\u001b[1;33mOptimization Complete\u001b[0m\u001b[33m \u001b[0m\u001b[33m════════════════════════════════════════════\u001b[0m\u001b[33m═╗\u001b[0m\n",
      "\u001b[33m║\u001b[0m                                                                                                                 \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[2mOptimizer:            \u001b[0m\u001b[2m \u001b[0m\u001b[1mFewShotBayesianOptimizer\u001b[0m                                                                 \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[2mModel Used:           \u001b[0m\u001b[2m \u001b[0mopenai/gpt-4o                                                                            \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[2mMetric Evaluated:     \u001b[0m\u001b[2m \u001b[0m\u001b[1mlevenshtein_ratio\u001b[0m                                                                        \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[2mInitial Score:        \u001b[0m\u001b[2m \u001b[0m0.1529                                                                                   \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[2mFinal Best Score:     \u001b[0m\u001b[2m \u001b[0m\u001b[1;36m0.4317\u001b[0m                                                                                   \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[2mTotal Improvement:    \u001b[0m\u001b[2m \u001b[0m\u001b[1;32m182.34%\u001b[0m                                                                                  \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[2mRounds Completed:     \u001b[0m\u001b[2m \u001b[0m0                                                                                        \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[2mOptimization run link:\u001b[0m\u001b[2m \u001b[0m\u001b]8;id=291476;https://www.comet.com/opik/api/v1/session/redirect/optimizations/?optimization_id=0198cd53-9f2d-7650-910e-d0bc21b526dd&dataset_id=0198ccfe-ec43-7649-97ad-fe00f0d2d788&path=aHR0cHM6Ly93d3cuY29tZXQuY29tL29waWsvYXBpLw==\u001b\\Open in Opik Dashboard\u001b]8;;\u001b\\                                                                   \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m                                                                                                                 \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m                                                                                                                 \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m╭─\u001b[0m\u001b[34m─────────────────────────────────────────\u001b[0m\u001b[34m \u001b[0m\u001b[1;34mFinal Optimized Prompt\u001b[0m\u001b[34m \u001b[0m\u001b[34m──────────────────────────────────────────\u001b[0m\u001b[34m─╮\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  \u001b[1;35mSystem:\u001b[0m Provide an answer to the question.                                                                 \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  ## Examples                                                                                                \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  ### Example                                                                                                \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Question: One Night in Istanbul stars a British actor and recording artist who is best known for his       \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  portrayal of Jonah Kirby on what BBC One drama?                                                            \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Answer: Waterloo Road                                                                                      \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  ### Example                                                                                                \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Question: Alicia Gräfin is best known for her role in a war film directed by who?                          \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Answer: David Ayer                                                                                         \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  ### Example                                                                                                \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Question: What comedy film from 1994 starred actress Heather Graham alongside James Le Gros?               \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Answer: Don't Do It                                                                                        \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  ### Example                                                                                                \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Question: Are Hüsker Dü and The Colourist both musical bands?                                              \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Answer: yes                                                                                                \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  ### Example                                                                                                \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Question: Who was the son of British radio and television presenter and historian best known as an         \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  analyst of election results?                                                                               \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Answer: Dan Snow                                                                                           \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  ### Example                                                                                                \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Question: Rectrix Aviation's jet charter business involves what service?                                   \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Answer: renting an entire aircraft                                                                         \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  ### Example                                                                                                \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Question: What U.S. court presided over both the Slaughter-House Cases and Reed v. Reed?                   \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  Answer: the Supreme Court                                                                                  \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  ---                                                                                                        \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  \u001b[1;32mUser:\u001b[0m {question}                                                                                           \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m  ---                                                                                                        \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m│\u001b[0m                                                                                                             \u001b[34m│\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m \u001b[34m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m \u001b[33m║\u001b[0m\n",
      "\u001b[33m║\u001b[0m                                                                                                                 \u001b[33m║\u001b[0m\n",
      "\u001b[33m╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝\u001b[0m\n",
      "Optimization run link: https://www.comet.com/opik/api/v1/session/redirect/optimizations/?optimization_id=0198cd53-9f2d-7650-910e-d0bc21b526dd&dataset_id=0198ccfe-ec43-7649-97ad-fe00f0d2d788&path=aHR0cHM6Ly93d3cuY29tZXQuY29tL29waWsvYXBpLw==\n"
     ]
    }
   ],
   "source": [
    "result.display()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "id": "5eWXMIGuVtn7",
    "outputId": "0b734df6-5885-41d6-84ee-02f3d8a18d40"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'https://www.comet.com/opik/api/v1/session/redirect/optimizations/?optimization_id=0198cd53-9f2d-7650-910e-d0bc21b526dd&dataset_id=0198ccfe-ec43-7649-97ad-fe00f0d2d788&path=aHR0cHM6Ly93d3cuY29tZXQuY29tL29waWsvYXBpLw=='"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.get_run_link()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1Zw5UhcCF3lm"
   },
   "source": [
    "Well done optimizer!\n",
    "\n",
    "You should see the percentage correct going from about 15% to about 50% (or more!) correct."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3_ppCUe6AGH8"
   },
   "source": [
    "What did we find? The result is a series of messages:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "o1nDwurG_2qL",
    "outputId": "17e7b683-d92b-42e5-92de-2c628eac8607"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'role': 'system',\n",
       "  'content': \"Provide an answer to the question.\\n\\n## Examples\\n\\n### Example\\n\\nQuestion: One Night in Istanbul stars a British actor and recording artist who is best known for his portrayal of Jonah Kirby on what BBC One drama?\\nAnswer: Waterloo Road\\n\\n\\n### Example\\n\\nQuestion: Alicia Gräfin is best known for her role in a war film directed by who?\\nAnswer: David Ayer\\n\\n\\n### Example\\n\\nQuestion: What comedy film from 1994 starred actress Heather Graham alongside James Le Gros?\\nAnswer: Don't Do It\\n\\n\\n### Example\\n\\nQuestion: Are Hüsker Dü and The Colourist both musical bands?\\nAnswer: yes\\n\\n\\n### Example\\n\\nQuestion: Who was the son of British radio and television presenter and historian best known as an analyst of election results?\\nAnswer: Dan Snow\\n\\n\\n### Example\\n\\nQuestion: Rectrix Aviation's jet charter business involves what service?\\nAnswer: renting an entire aircraft\\n\\n\\n### Example\\n\\nQuestion: What U.S. court presided over both the Slaughter-House Cases and Reed v. Reed?\\nAnswer: the Supreme Court\\n\"},\n",
       " {'role': 'user', 'content': '{question}'}]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.details[\"chat_messages\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bENwStbQ-wEv"
   },
   "source": [
    "## Opik Visualization UI\n",
    "\n",
    "When you create an Optimization Run, you'll see it in the Opik UI (either running locally or hosted).\n",
    "\n",
    "If you go to your `comet.com/opik/YOURID/optimizations` page, you'll see your run at the top of the list:\n",
    "\n",
    "<img src=\"https://raw.githubusercontent.com/comet-ml/opik/refs/heads/main/sdks/opik_optimizer/docs/images/optimizer-ui.png\" width=\"500px\">\n",
    "\n",
    "Along the top of the page you'll see a running history of the metric scores, with the latest dataset selected.\n",
    "\n",
    "If you click on your run, you'll see the set of trials that ran durring this optimization run. Running across the top of this page are the scores just for this trial. On the top right you'll see the best so far:\n",
    "\n",
    "<img src=\"https://raw.githubusercontent.com/comet-ml/opik/refs/heads/main/sdks/opik_optimizer/docs/images/optimize-trials.png\" width=\"500px\">\n",
    "\n",
    "If you click on a trial, you'll see the prompt for that trial:\n",
    "\n",
    "<img src=\"https://raw.githubusercontent.com/comet-ml/opik/refs/heads/main/sdks/opik_optimizer/docs/images/optimizer-prompt.png\" width=\"500px\">\n",
    "\n",
    "From the trial page you can also see the Trial items, and even dig down (mouse over the \"Evaluation task\" column on a row) to see the traces."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "J7bVb_Pp9TUb"
   },
   "source": [
    "## Using Optimized Prompts\n",
    "\n",
    "How can we use the optimized results?\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6uWZEjWTFcpJ"
   },
   "source": [
    "Once we have the \"chat_messages\", we can do the following:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "-wOD7oca9gjs"
   },
   "outputs": [],
   "source": [
    "from litellm.integrations.opik.opik import OpikLogger\n",
    "import litellm\n",
    "\n",
    "opik_logger = OpikLogger()\n",
    "litellm.callbacks = [opik_logger]\n",
    "\n",
    "\n",
    "def query(question, chat_messages):\n",
    "    messages = chat_messages[:-1]  # Cut off the last one\n",
    "    # replace it with question in proper format:\n",
    "    messages.append({\"role\": \"user\", \"content\": '{\"question\": \"%s\"}\"}' % question})\n",
    "\n",
    "    response = litellm.completion(\n",
    "        model=\"gpt-4o-mini\",\n",
    "        temperature=0.1,\n",
    "        max_tokens=5000,\n",
    "        messages=messages,\n",
    "    )\n",
    "    return response.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "id": "2nu5Tqdl9wa8",
    "outputId": "970e8c50-cd50-4fe2-efa6-307f8c666b21"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'Answer: April 20, 1966'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query(\"When was David Chalmers born?\", result.details[\"chat_messages\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "id": "xI3mOXQ4_btf",
    "outputId": "ff7b4718-45c9-40d4-d8c4-448852cbde54"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'Answer: a baby elephant'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query(\"What weighs more: a baby elephant or an SUV?\", result.details[\"chat_messages\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2x0JkiYhFsQq"
   },
   "source": [
    "If it says \"elephant\" that is not correct!\n",
    "\n",
    "We'll need to use an agent with tools to get a better answer."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uSDJ1bFx51kd"
   },
   "source": [
    "# Next Steps\n",
    "\n",
    "You can try out other optimizers. More details can be found in the [Opik Agent Optimizer documentation](https://www.comet.com/docs/opik/agent_optimization/overview)."
   ]
  }
 ],
 "metadata": {
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}