wehub-resource-sync
a0c8464e58
Build Package / build (ubuntu-latest) (push) Failing after 1s
CodeQL / Analyze (python) (push) Failing after 1s
Core Typecheck / core-typecheck (push) Failing after 1s
Linting / lint (push) Failing after 1s
llama-dev tests / test-llama-dev (push) Failing after 1s
Publish Sub-Package to PyPI if Needed / publish_subpackage_if_needed (push) Has been skipped
Sync Docs to Developer Hub / sync-docs (push) Failing after 0s
Build Package / build (windows-latest) (push) Has been cancelled
722 lines
41 KiB
Plaintext
722 lines
41 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "0c266183",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Relevancy Evaluator\n",
|
||
"\n",
|
||
"This notebook uses the `RelevancyEvaluator` to measure if the response + source nodes match the query. \n",
|
||
"This is useful for measuring if the query was actually answered by the response."
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "36dadd9e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"%pip install llama-index-llms-openai pandas[jinja2] spacy"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "9080b39e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import logging\n",
|
||
"import sys\n",
|
||
"\n",
|
||
"logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n",
|
||
"logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "8d0b2364-4806-4656-81e7-3f6e4b910b5b",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"from llama_index.core import (\n",
|
||
" TreeIndex,\n",
|
||
" VectorStoreIndex,\n",
|
||
" SimpleDirectoryReader,\n",
|
||
" Response,\n",
|
||
")\n",
|
||
"from llama_index.llms.openai import OpenAI\n",
|
||
"from llama_index.core.evaluation import RelevancyEvaluator\n",
|
||
"from llama_index.core.node_parser import SentenceSplitter\n",
|
||
"import pandas as pd\n",
|
||
"\n",
|
||
"pd.set_option(\"display.max_colwidth\", 0)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "b9b98f89-d5b8-4d29-92f6-ad76d5060e9f",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# gpt-3 (davinci)\n",
|
||
"gpt3 = OpenAI(temperature=0, model=\"gpt-3.5-turbo\")\n",
|
||
"\n",
|
||
"# gpt-4\n",
|
||
"gpt4 = OpenAI(temperature=0, model=\"gpt-4\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "8eb3e616-64e5-4bf4-a67b-661e9b3657e7",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"evaluator = RelevancyEvaluator(llm=gpt3)\n",
|
||
"evaluator_gpt4 = RelevancyEvaluator(llm=gpt4)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "1298bbb4-c99e-431e-93ef-eb32c0a2fc2a",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"documents = SimpleDirectoryReader(\"./test_wiki_data\").load_data()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "41f0e53f-77a6-40d5-94ae-3f81b01af75c",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# create vector index\n",
|
||
"splitter = SentenceSplitter(chunk_size=512)\n",
|
||
"vector_index = VectorStoreIndex.from_documents(\n",
|
||
" documents, transformations=[splitter]\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "af730b2e-6949-4865-b7af-bb2bc60a9173",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"from llama_index.core.evaluation import EvaluationResult\n",
|
||
"\n",
|
||
"\n",
|
||
"# define jupyter display function\n",
|
||
"def display_eval_df(\n",
|
||
" query: str, response: Response, eval_result: EvaluationResult\n",
|
||
") -> None:\n",
|
||
" eval_df = pd.DataFrame(\n",
|
||
" {\n",
|
||
" \"Query\": query,\n",
|
||
" \"Response\": str(response),\n",
|
||
" \"Source\": response.source_nodes[0].node.text[:1000] + \"...\",\n",
|
||
" \"Evaluation Result\": \"Pass\" if eval_result.passing else \"Fail\",\n",
|
||
" \"Reasoning\": eval_result.feedback,\n",
|
||
" },\n",
|
||
" index=[0],\n",
|
||
" )\n",
|
||
" eval_df = eval_df.style.set_properties(\n",
|
||
" **{\n",
|
||
" \"inline-size\": \"600px\",\n",
|
||
" \"overflow-wrap\": \"break-word\",\n",
|
||
" },\n",
|
||
" subset=[\"Response\", \"Source\"],\n",
|
||
" )\n",
|
||
" display(eval_df)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "4780e16a-aa6c-4143-978d-4a93a4357130",
|
||
"metadata": {},
|
||
"source": [
|
||
"### Evaluate Response\n",
|
||
"\n",
|
||
"Evaluate response relative to source nodes as well as query."
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "180a5d2e-9286-477b-9cd0-a5976d18d845",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"query_str = (\n",
|
||
" \"What battles took place in New York City in the American Revolution?\"\n",
|
||
")\n",
|
||
"query_engine = vector_index.as_query_engine()\n",
|
||
"response_vector = query_engine.query(query_str)\n",
|
||
"eval_result = evaluator_gpt4.evaluate_response(\n",
|
||
" query=query_str, response=response_vector\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c764b8b3-69b1-4ac8-b88b-3f9e204b8bfb",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<style type=\"text/css\">\n",
|
||
"#T_84181_row0_col1, #T_84181_row0_col2 {\n",
|
||
" inline-size: 600px;\n",
|
||
" overflow-wrap: break-word;\n",
|
||
"}\n",
|
||
"</style>\n",
|
||
"<table id=\"T_84181\">\n",
|
||
" <thead>\n",
|
||
" <tr>\n",
|
||
" <th class=\"blank level0\" > </th>\n",
|
||
" <th id=\"T_84181_level0_col0\" class=\"col_heading level0 col0\" >Query</th>\n",
|
||
" <th id=\"T_84181_level0_col1\" class=\"col_heading level0 col1\" >Response</th>\n",
|
||
" <th id=\"T_84181_level0_col2\" class=\"col_heading level0 col2\" >Source</th>\n",
|
||
" <th id=\"T_84181_level0_col3\" class=\"col_heading level0 col3\" >Evaluation Result</th>\n",
|
||
" <th id=\"T_84181_level0_col4\" class=\"col_heading level0 col4\" >Reasoning</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th id=\"T_84181_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
|
||
" <td id=\"T_84181_row0_col0\" class=\"data row0 col0\" >What battles took place in New York City in the American Revolution?</td>\n",
|
||
" <td id=\"T_84181_row0_col1\" class=\"data row0 col1\" >The Battle of Long Island was the largest battle of the American Revolutionary War that took place in New York City.</td>\n",
|
||
" <td id=\"T_84181_row0_col2\" class=\"data row0 col2\" >=== American Revolution ===\n",
|
||
"\n",
|
||
"The Stamp Act Congress met in New York in October 1765, as the Sons of Liberty organization emerged in the city and skirmished over the next ten years with British troops stationed there. The Battle of Long Island, the largest battle of the American Revolutionary War, was fought in August 1776 within the modern-day borough of Brooklyn. After the battle, in which the Americans were defeated, the British made the city their military and political base of operations in North America. The city was a haven for Loyalist refugees and escaped slaves who joined the British lines for freedom newly promised by the Crown for all fighters. As many as 10,000 escaped slaves crowded into the city during the British occupation. When the British forces evacuated at the close of the war in 1783, they transported 3,000 freedmen for resettlement in Nova Scotia. They resettled other freedmen in England and the Caribbean.\n",
|
||
"The only attempt at a peaceful solution to the war took pl...</td>\n",
|
||
" <td id=\"T_84181_row0_col3\" class=\"data row0 col3\" >Pass</td>\n",
|
||
" <td id=\"T_84181_row0_col4\" class=\"data row0 col4\" >The context confirms that the Battle of Long Island, which was the largest battle of the American Revolutionary War, took place in New York City.</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n"
|
||
],
|
||
"text/plain": [
|
||
"<pandas.io.formats.style.Styler at 0x141697dd0>"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
}
|
||
],
|
||
"source": [
|
||
"display_eval_df(query_str, response_vector, eval_result)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "97f3ddf1-8dc2-4fb8-831f-2c06649e0955",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"query_str = \"What are the airports in New York City?\"\n",
|
||
"query_engine = vector_index.as_query_engine()\n",
|
||
"response_vector = query_engine.query(query_str)\n",
|
||
"eval_result = evaluator_gpt4.evaluate_response(\n",
|
||
" query=query_str, response=response_vector\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "01c53014-82b0-4865-b849-c0beb042143d",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<style type=\"text/css\">\n",
|
||
"#T_55f60_row0_col1, #T_55f60_row0_col2 {\n",
|
||
" inline-size: 600px;\n",
|
||
" overflow-wrap: break-word;\n",
|
||
"}\n",
|
||
"</style>\n",
|
||
"<table id=\"T_55f60\">\n",
|
||
" <thead>\n",
|
||
" <tr>\n",
|
||
" <th class=\"blank level0\" > </th>\n",
|
||
" <th id=\"T_55f60_level0_col0\" class=\"col_heading level0 col0\" >Query</th>\n",
|
||
" <th id=\"T_55f60_level0_col1\" class=\"col_heading level0 col1\" >Response</th>\n",
|
||
" <th id=\"T_55f60_level0_col2\" class=\"col_heading level0 col2\" >Source</th>\n",
|
||
" <th id=\"T_55f60_level0_col3\" class=\"col_heading level0 col3\" >Evaluation Result</th>\n",
|
||
" <th id=\"T_55f60_level0_col4\" class=\"col_heading level0 col4\" >Reasoning</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th id=\"T_55f60_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
|
||
" <td id=\"T_55f60_row0_col0\" class=\"data row0 col0\" >What are the airports in New York City?</td>\n",
|
||
" <td id=\"T_55f60_row0_col1\" class=\"data row0 col1\" >The airports in New York City include John F. Kennedy International Airport, Newark Liberty International Airport, LaGuardia Airport, Stewart International Airport, Long Island MacArthur Airport, Trenton-Mercer Airport, and Westchester County Airport.</td>\n",
|
||
" <td id=\"T_55f60_row0_col2\" class=\"data row0 col2\" >along the Northeast Corridor, and long-distance train service to other North American cities.The Staten Island Railway rapid transit system solely serves Staten Island, operating 24 hours a day. The Port Authority Trans-Hudson (PATH train) links Midtown and Lower Manhattan to northeastern New Jersey, primarily Hoboken, Jersey City, and Newark. Like the New York City Subway, the PATH operates 24 hours a day; meaning three of the six rapid transit systems in the world which operate on 24-hour schedules are wholly or partly in New York (the others are a portion of the Chicago \"L\", the PATCO Speedline serving Philadelphia, and the Copenhagen Metro).\n",
|
||
"Multibillion-dollar heavy rail transit projects under construction in New York City include the Second Avenue Subway, and the East Side Access project.\n",
|
||
"\n",
|
||
"\n",
|
||
"==== Buses ====\n",
|
||
"\n",
|
||
"New York City's public bus fleet runs 24/7 and is the largest in North America. The Port Authority Bus Terminal, the main intercity bus terminal of the city, serves 7,000 buse...</td>\n",
|
||
" <td id=\"T_55f60_row0_col3\" class=\"data row0 col3\" >Pass</td>\n",
|
||
" <td id=\"T_55f60_row0_col4\" class=\"data row0 col4\" >The context provides information about the airports in New York City, which includes John F. Kennedy International Airport, Newark Liberty International Airport, LaGuardia Airport, Stewart International Airport, Long Island MacArthur Airport, Trenton-Mercer Airport, and Westchester County Airport. This matches the response to the query.</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n"
|
||
],
|
||
"text/plain": [
|
||
"<pandas.io.formats.style.Styler at 0x140ac2e10>"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
}
|
||
],
|
||
"source": [
|
||
"display_eval_df(query_str, response_vector, eval_result)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "8b860691-9f6a-4b2f-bfef-5a5a56693a18",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"query_str = \"Who is the mayor of New York City?\"\n",
|
||
"query_engine = vector_index.as_query_engine()\n",
|
||
"response_vector = query_engine.query(query_str)\n",
|
||
"eval_result = evaluator_gpt4.evaluate_response(\n",
|
||
" query=query_str, response=response_vector\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "91cce4e3-b4e5-4583-9543-6c3726c0e7d3",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<style type=\"text/css\">\n",
|
||
"#T_1c9c8_row0_col1, #T_1c9c8_row0_col2 {\n",
|
||
" inline-size: 600px;\n",
|
||
" overflow-wrap: break-word;\n",
|
||
"}\n",
|
||
"</style>\n",
|
||
"<table id=\"T_1c9c8\">\n",
|
||
" <thead>\n",
|
||
" <tr>\n",
|
||
" <th class=\"blank level0\" > </th>\n",
|
||
" <th id=\"T_1c9c8_level0_col0\" class=\"col_heading level0 col0\" >Query</th>\n",
|
||
" <th id=\"T_1c9c8_level0_col1\" class=\"col_heading level0 col1\" >Response</th>\n",
|
||
" <th id=\"T_1c9c8_level0_col2\" class=\"col_heading level0 col2\" >Source</th>\n",
|
||
" <th id=\"T_1c9c8_level0_col3\" class=\"col_heading level0 col3\" >Evaluation Result</th>\n",
|
||
" <th id=\"T_1c9c8_level0_col4\" class=\"col_heading level0 col4\" >Reasoning</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th id=\"T_1c9c8_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
|
||
" <td id=\"T_1c9c8_row0_col0\" class=\"data row0 col0\" >Who is the mayor of New York City?</td>\n",
|
||
" <td id=\"T_1c9c8_row0_col1\" class=\"data row0 col1\" >The mayor of New York City is Eric Adams.</td>\n",
|
||
" <td id=\"T_1c9c8_row0_col2\" class=\"data row0 col2\" >=== Politics ===\n",
|
||
"The present mayor is Eric Adams. He was elected in 2021 with 67% of the vote, and assumed office on January 1, 2022.\n",
|
||
"The Democratic Party holds the majority of public offices. As of April 2016, 69% of registered voters in the city are Democrats and 10% are Republicans. New York City has not been carried by a Republican presidential election since President Calvin Coolidge won the five boroughs in 1924. A Republican candidate for statewide office has not won all five boroughs of the city since it was incorporated in 1898. In 2012, Democrat Barack Obama became the first presidential candidate of any party to receive more than 80% of the overall vote in New York City, sweeping all five boroughs. Party platforms center on affordable housing, education, and economic development, and labor politics are of importance in the city. Thirteen out of 27 U.S. congressional districts in the state of New York include portions of New York City.New York is one of the most important so...</td>\n",
|
||
" <td id=\"T_1c9c8_row0_col3\" class=\"data row0 col3\" >Pass</td>\n",
|
||
" <td id=\"T_1c9c8_row0_col4\" class=\"data row0 col4\" >The context confirms that Eric Adams is the current mayor of New York City, as stated in the response.</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n"
|
||
],
|
||
"text/plain": [
|
||
"<pandas.io.formats.style.Styler at 0x1416ecc10>"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
}
|
||
],
|
||
"source": [
|
||
"display_eval_df(query_str, response_vector, eval_result)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "0ee6a336-8fd0-46b3-bb8f-7f47a8781c60",
|
||
"metadata": {},
|
||
"source": [
|
||
"### Evaluate Source Nodes\n",
|
||
"\n",
|
||
"Evaluate the set of returned sources, and determine which sources actually contain the answer to a given query."
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "74eed7ba-7ed5-44fa-af4d-356f6ce3b710",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"from typing import List\n",
|
||
"\n",
|
||
"\n",
|
||
"# define jupyter display function\n",
|
||
"def display_eval_sources(\n",
|
||
" query: str, response: Response, eval_result: List[str]\n",
|
||
") -> None:\n",
|
||
" sources = [s.node.get_text() for s in response.source_nodes]\n",
|
||
" eval_df = pd.DataFrame(\n",
|
||
" {\n",
|
||
" \"Source\": sources,\n",
|
||
" \"Eval Result\": eval_result,\n",
|
||
" },\n",
|
||
" )\n",
|
||
" eval_df.style.set_caption(query)\n",
|
||
" eval_df = eval_df.style.set_properties(\n",
|
||
" **{\n",
|
||
" \"inline-size\": \"600px\",\n",
|
||
" \"overflow-wrap\": \"break-word\",\n",
|
||
" },\n",
|
||
" subset=[\"Source\"],\n",
|
||
" )\n",
|
||
"\n",
|
||
" display(eval_df)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "0e3674ed-2632-4f48-9102-895fea94e340",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# NOTE: you can set response_mode=\"no_text\" to get just the sources\n",
|
||
"query_str = \"What are the airports in New York City?\"\n",
|
||
"query_engine = vector_index.as_query_engine(\n",
|
||
" similarity_top_k=3, response_mode=\"no_text\"\n",
|
||
")\n",
|
||
"response_vector = query_engine.query(query_str)\n",
|
||
"eval_source_result_full = [\n",
|
||
" evaluator_gpt4.evaluate(\n",
|
||
" query=query_str,\n",
|
||
" response=response_vector.response,\n",
|
||
" contexts=[source_node.get_content()],\n",
|
||
" )\n",
|
||
" for source_node in response_vector.source_nodes\n",
|
||
"]\n",
|
||
"eval_source_result = [\n",
|
||
" \"Pass\" if result.passing else \"Fail\" for result in eval_source_result_full\n",
|
||
"]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "35125a2d-927e-40d3-ac83-da9297b6e81e",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<style type=\"text/css\">\n",
|
||
"#T_f4156_row0_col0, #T_f4156_row1_col0, #T_f4156_row2_col0 {\n",
|
||
" inline-size: 600px;\n",
|
||
" overflow-wrap: break-word;\n",
|
||
"}\n",
|
||
"</style>\n",
|
||
"<table id=\"T_f4156\">\n",
|
||
" <thead>\n",
|
||
" <tr>\n",
|
||
" <th class=\"blank level0\" > </th>\n",
|
||
" <th id=\"T_f4156_level0_col0\" class=\"col_heading level0 col0\" >Source</th>\n",
|
||
" <th id=\"T_f4156_level0_col1\" class=\"col_heading level0 col1\" >Eval Result</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th id=\"T_f4156_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
|
||
" <td id=\"T_f4156_row0_col0\" class=\"data row0 col0\" >along the Northeast Corridor, and long-distance train service to other North American cities.The Staten Island Railway rapid transit system solely serves Staten Island, operating 24 hours a day. The Port Authority Trans-Hudson (PATH train) links Midtown and Lower Manhattan to northeastern New Jersey, primarily Hoboken, Jersey City, and Newark. Like the New York City Subway, the PATH operates 24 hours a day; meaning three of the six rapid transit systems in the world which operate on 24-hour schedules are wholly or partly in New York (the others are a portion of the Chicago \"L\", the PATCO Speedline serving Philadelphia, and the Copenhagen Metro).\n",
|
||
"Multibillion-dollar heavy rail transit projects under construction in New York City include the Second Avenue Subway, and the East Side Access project.\n",
|
||
"\n",
|
||
"\n",
|
||
"==== Buses ====\n",
|
||
"\n",
|
||
"New York City's public bus fleet runs 24/7 and is the largest in North America. The Port Authority Bus Terminal, the main intercity bus terminal of the city, serves 7,000 buses and 200,000 commuters daily, making it the busiest bus station in the world.\n",
|
||
"\n",
|
||
"\n",
|
||
"=== Air ===\n",
|
||
"\n",
|
||
"New York's airspace is the busiest in the United States and one of the world's busiest air transportation corridors. The three busiest airports in the New York metropolitan area include John F. Kennedy International Airport, Newark Liberty International Airport, and LaGuardia Airport; 130.5 million travelers used these three airports in 2016. JFK and Newark Liberty were the busiest and fourth busiest U.S. gateways for international air passengers, respectively, in 2012; as of 2011, JFK was the busiest airport for international passengers in North America.Plans have advanced to expand passenger volume at a fourth airport, Stewart International Airport near Newburgh, New York, by the Port Authority of New York and New Jersey. Plans were announced in July 2015 to entirely rebuild LaGuardia Airport in a multibillion-dollar project to replace its aging facilities. Other commercial airports in or serving the New York metropolitan area include Long Island MacArthur Airport, Trenton–Mercer Airport and Westchester County Airport. The primary general aviation airport serving the area is Teterboro Airport.</td>\n",
|
||
" <td id=\"T_f4156_row0_col1\" class=\"data row0 col1\" >Pass</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th id=\"T_f4156_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
|
||
" <td id=\"T_f4156_row1_col0\" class=\"data row1 col0\" >See or edit raw graph data.\n",
|
||
"\n",
|
||
"\n",
|
||
"=== Parks ===\n",
|
||
"\n",
|
||
"The city of New York has a complex park system, with various lands operated by the National Park Service, the New York State Office of Parks, Recreation and Historic Preservation, and the New York City Department of Parks and Recreation. In its 2018 ParkScore ranking, the Trust for Public Land reported that the park system in New York City was the ninth-best park system among the fifty most populous U.S. cities. ParkScore ranks urban park systems by a formula that analyzes median park size, park acres as percent of city area, the percent of city residents within a half-mile of a park, spending of park services per resident, and the number of playgrounds per 10,000 residents. In 2021, the New York City Council banned the use of synthetic pesticides by city agencies and instead required organic lawn management. The effort was started by teacher Paula Rogovin's kindergarten class at P.S. 290.\n",
|
||
"\n",
|
||
"\n",
|
||
"==== National parks ====\n",
|
||
"\n",
|
||
"Gateway National Recreation Area contains over 26,000 acres (110 km2), most of it in New York City. In Brooklyn and Queens, the park contains over 9,000 acres (36 km2) of salt marsh, wetlands, islands, and water, including most of Jamaica Bay and the Jamaica Bay Wildlife Refuge. Also in Queens, the park includes a significant portion of the western Rockaway Peninsula, most notably Jacob Riis Park and Fort Tilden. In Staten Island, it includes Fort Wadsworth, with historic pre-Civil War era Battery Weed and Fort Tompkins, and Great Kills Park, with beaches, trails, and a marina.\n",
|
||
"The Statue of Liberty National Monument and Ellis Island Immigration Museum are managed by the National Park Service and are in both New York and New Jersey. They are joined in the harbor by Governors Island National Monument. Historic sites under federal management on Manhattan Island include Stonewall National Monument; Castle Clinton National Monument; Federal Hall National Memorial; Theodore Roosevelt Birthplace National Historic Site; General Grant National Memorial (Grant's Tomb); African Burial Ground National Monument; and Hamilton Grange National Memorial. Hundreds of properties are listed on the National Register of Historic Places or as a National Historic Landmark.</td>\n",
|
||
" <td id=\"T_f4156_row1_col1\" class=\"data row1 col1\" >Fail</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th id=\"T_f4156_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
|
||
" <td id=\"T_f4156_row2_col0\" class=\"data row2 col0\" >New York has witnessed a growing combined volume of international and domestic tourists, reflecting over 60 million visitors to the city per year, the world's busiest tourist destination. Approximately 12 million visitors to New York City have been from outside the United States, with the highest numbers from the United Kingdom, Canada, Brazil, and China. Multiple sources have called New York the most photographed city in the world.I Love New York (stylized I ❤ NY) is both a logo and a song that are the basis of an advertising campaign and have been used since 1977 to promote tourism in New York City, and later to promote New York State as well. The trademarked logo, owned by New York State Empire State Development, appears in souvenir shops and brochures throughout the city and state, some licensed, many not. The song is the state song of New York.\n",
|
||
"The majority of the most high-profile tourist destinations to the city are situated in Manhattan. These include Times Square; Broadway theater productions; the Empire State Building; the Statue of Liberty; Ellis Island; the United Nations headquarters; the World Trade Center (including the National September 11 Memorial & Museum and One World Trade Center); the art museums along Museum Mile; green spaces such as Central Park, Washington Square Park, the High Line, and the medieval gardens of The Cloisters; the Stonewall Inn; Rockefeller Center; ethnic enclaves including the Manhattan Chinatown, Koreatown, Curry Hill, Harlem, Spanish Harlem, Little Italy, and Little Australia; luxury shopping along Fifth and Madison Avenues; and events such as the Halloween Parade in Greenwich Village; the Brooklyn Bridge (shared with Brooklyn); the Macy's Thanksgiving Day Parade; the lighting of the Rockefeller Center Christmas Tree; the St. Patrick's Day Parade; seasonal activities such as ice skating in Central Park in the wintertime; the Tribeca Film Festival; and free performances in Central Park at SummerStage.Points of interest have also developed in the city outside Manhattan and have made the outer boroughs tourist destinations in their own right.</td>\n",
|
||
" <td id=\"T_f4156_row2_col1\" class=\"data row2 col1\" >Fail</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n"
|
||
],
|
||
"text/plain": [
|
||
"<pandas.io.formats.style.Styler at 0x1416b0e90>"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
}
|
||
],
|
||
"source": [
|
||
"display_eval_sources(query_str, response_vector, eval_source_result)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "cdc340a9-1853-414e-9e87-820aa5d82db7",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# NOTE: you can set response_mode=\"no_text\" to get just the sources\n",
|
||
"query_str = \"Who is the mayor of New York City?\"\n",
|
||
"query_engine = vector_index.as_query_engine(\n",
|
||
" similarity_top_k=3, response_mode=\"no_text\"\n",
|
||
")\n",
|
||
"eval_source_result_full = [\n",
|
||
" evaluator_gpt4.evaluate(\n",
|
||
" query=query_str,\n",
|
||
" response=response_vector.response,\n",
|
||
" contexts=[source_node.get_content()],\n",
|
||
" )\n",
|
||
" for source_node in response_vector.source_nodes\n",
|
||
"]\n",
|
||
"eval_source_result = [\n",
|
||
" \"Pass\" if result.passing else \"Fail\" for result in eval_source_result_full\n",
|
||
"]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "b88eb350-cae6-4890-9b55-cb5f750afa9e",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<style type=\"text/css\">\n",
|
||
"#T_3fd29_row0_col0, #T_3fd29_row1_col0, #T_3fd29_row2_col0 {\n",
|
||
" inline-size: 600px;\n",
|
||
" overflow-wrap: break-word;\n",
|
||
"}\n",
|
||
"</style>\n",
|
||
"<table id=\"T_3fd29\">\n",
|
||
" <thead>\n",
|
||
" <tr>\n",
|
||
" <th class=\"blank level0\" > </th>\n",
|
||
" <th id=\"T_3fd29_level0_col0\" class=\"col_heading level0 col0\" >Source</th>\n",
|
||
" <th id=\"T_3fd29_level0_col1\" class=\"col_heading level0 col1\" >Eval Result</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th id=\"T_3fd29_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
|
||
" <td id=\"T_3fd29_row0_col0\" class=\"data row0 col0\" >along the Northeast Corridor, and long-distance train service to other North American cities.The Staten Island Railway rapid transit system solely serves Staten Island, operating 24 hours a day. The Port Authority Trans-Hudson (PATH train) links Midtown and Lower Manhattan to northeastern New Jersey, primarily Hoboken, Jersey City, and Newark. Like the New York City Subway, the PATH operates 24 hours a day; meaning three of the six rapid transit systems in the world which operate on 24-hour schedules are wholly or partly in New York (the others are a portion of the Chicago \"L\", the PATCO Speedline serving Philadelphia, and the Copenhagen Metro).\n",
|
||
"Multibillion-dollar heavy rail transit projects under construction in New York City include the Second Avenue Subway, and the East Side Access project.\n",
|
||
"\n",
|
||
"\n",
|
||
"==== Buses ====\n",
|
||
"\n",
|
||
"New York City's public bus fleet runs 24/7 and is the largest in North America. The Port Authority Bus Terminal, the main intercity bus terminal of the city, serves 7,000 buses and 200,000 commuters daily, making it the busiest bus station in the world.\n",
|
||
"\n",
|
||
"\n",
|
||
"=== Air ===\n",
|
||
"\n",
|
||
"New York's airspace is the busiest in the United States and one of the world's busiest air transportation corridors. The three busiest airports in the New York metropolitan area include John F. Kennedy International Airport, Newark Liberty International Airport, and LaGuardia Airport; 130.5 million travelers used these three airports in 2016. JFK and Newark Liberty were the busiest and fourth busiest U.S. gateways for international air passengers, respectively, in 2012; as of 2011, JFK was the busiest airport for international passengers in North America.Plans have advanced to expand passenger volume at a fourth airport, Stewart International Airport near Newburgh, New York, by the Port Authority of New York and New Jersey. Plans were announced in July 2015 to entirely rebuild LaGuardia Airport in a multibillion-dollar project to replace its aging facilities. Other commercial airports in or serving the New York metropolitan area include Long Island MacArthur Airport, Trenton–Mercer Airport and Westchester County Airport. The primary general aviation airport serving the area is Teterboro Airport.</td>\n",
|
||
" <td id=\"T_3fd29_row0_col1\" class=\"data row0 col1\" >Fail</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th id=\"T_3fd29_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
|
||
" <td id=\"T_3fd29_row1_col0\" class=\"data row1 col0\" >See or edit raw graph data.\n",
|
||
"\n",
|
||
"\n",
|
||
"=== Parks ===\n",
|
||
"\n",
|
||
"The city of New York has a complex park system, with various lands operated by the National Park Service, the New York State Office of Parks, Recreation and Historic Preservation, and the New York City Department of Parks and Recreation. In its 2018 ParkScore ranking, the Trust for Public Land reported that the park system in New York City was the ninth-best park system among the fifty most populous U.S. cities. ParkScore ranks urban park systems by a formula that analyzes median park size, park acres as percent of city area, the percent of city residents within a half-mile of a park, spending of park services per resident, and the number of playgrounds per 10,000 residents. In 2021, the New York City Council banned the use of synthetic pesticides by city agencies and instead required organic lawn management. The effort was started by teacher Paula Rogovin's kindergarten class at P.S. 290.\n",
|
||
"\n",
|
||
"\n",
|
||
"==== National parks ====\n",
|
||
"\n",
|
||
"Gateway National Recreation Area contains over 26,000 acres (110 km2), most of it in New York City. In Brooklyn and Queens, the park contains over 9,000 acres (36 km2) of salt marsh, wetlands, islands, and water, including most of Jamaica Bay and the Jamaica Bay Wildlife Refuge. Also in Queens, the park includes a significant portion of the western Rockaway Peninsula, most notably Jacob Riis Park and Fort Tilden. In Staten Island, it includes Fort Wadsworth, with historic pre-Civil War era Battery Weed and Fort Tompkins, and Great Kills Park, with beaches, trails, and a marina.\n",
|
||
"The Statue of Liberty National Monument and Ellis Island Immigration Museum are managed by the National Park Service and are in both New York and New Jersey. They are joined in the harbor by Governors Island National Monument. Historic sites under federal management on Manhattan Island include Stonewall National Monument; Castle Clinton National Monument; Federal Hall National Memorial; Theodore Roosevelt Birthplace National Historic Site; General Grant National Memorial (Grant's Tomb); African Burial Ground National Monument; and Hamilton Grange National Memorial. Hundreds of properties are listed on the National Register of Historic Places or as a National Historic Landmark.</td>\n",
|
||
" <td id=\"T_3fd29_row1_col1\" class=\"data row1 col1\" >Fail</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th id=\"T_3fd29_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
|
||
" <td id=\"T_3fd29_row2_col0\" class=\"data row2 col0\" >New York has witnessed a growing combined volume of international and domestic tourists, reflecting over 60 million visitors to the city per year, the world's busiest tourist destination. Approximately 12 million visitors to New York City have been from outside the United States, with the highest numbers from the United Kingdom, Canada, Brazil, and China. Multiple sources have called New York the most photographed city in the world.I Love New York (stylized I ❤ NY) is both a logo and a song that are the basis of an advertising campaign and have been used since 1977 to promote tourism in New York City, and later to promote New York State as well. The trademarked logo, owned by New York State Empire State Development, appears in souvenir shops and brochures throughout the city and state, some licensed, many not. The song is the state song of New York.\n",
|
||
"The majority of the most high-profile tourist destinations to the city are situated in Manhattan. These include Times Square; Broadway theater productions; the Empire State Building; the Statue of Liberty; Ellis Island; the United Nations headquarters; the World Trade Center (including the National September 11 Memorial & Museum and One World Trade Center); the art museums along Museum Mile; green spaces such as Central Park, Washington Square Park, the High Line, and the medieval gardens of The Cloisters; the Stonewall Inn; Rockefeller Center; ethnic enclaves including the Manhattan Chinatown, Koreatown, Curry Hill, Harlem, Spanish Harlem, Little Italy, and Little Australia; luxury shopping along Fifth and Madison Avenues; and events such as the Halloween Parade in Greenwich Village; the Brooklyn Bridge (shared with Brooklyn); the Macy's Thanksgiving Day Parade; the lighting of the Rockefeller Center Christmas Tree; the St. Patrick's Day Parade; seasonal activities such as ice skating in Central Park in the wintertime; the Tribeca Film Festival; and free performances in Central Park at SummerStage.Points of interest have also developed in the city outside Manhattan and have made the outer boroughs tourist destinations in their own right.</td>\n",
|
||
" <td id=\"T_3fd29_row2_col1\" class=\"data row2 col1\" >Fail</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n"
|
||
],
|
||
"text/plain": [
|
||
"<pandas.io.formats.style.Styler at 0x1418264d0>"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
}
|
||
],
|
||
"source": [
|
||
"display_eval_sources(query_str, response_vector, eval_source_result)"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|