wehub-resource-sync
302 lines
8.1 KiB
Plaintext
302 lines
8.1 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "860c9e4b-dc7c-4f2e-8f60-96cccf61d43c",
|
|
"metadata": {},
|
|
"source": [
|
|
"# OpenLayer\n",
|
|
"## Evaluating RAG pipelines with Openlayer and Ragas\n",
|
|
"\n",
|
|
"[Openlayer](https://www.openlayer.com/) is an evaluation tool that fits into your development and production pipelines to help you ship high-quality models with confidence.\n",
|
|
"\n",
|
|
"This notebook should be used together with [this blog post](https://www.openlayer.com/blog/post/evaluating-rag-pipelines-with-ragas-and-openlayer)."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "3ad3ed0c-e495-4078-ab95-a70fa6322ab1",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Pre-requisites"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7ded5103-b6ac-482e-9217-347f701333b4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%%bash\n",
|
|
"git clone https://huggingface.co/datasets/vibrantlabsai/prompt-engineering-papers"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "58f0951f-5de9-4eca-8b0c-e77d5ac99bad",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import os\n",
|
|
"\n",
|
|
"os.environ[\"OPENAI_API_KEY\"] = \"YOUR_OPENAI_API_KEY_HERE\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "93b95703-0826-47b2-8b0b-e0f982b1e170",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Synthetic test data generation"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "69cfc916-148a-4608-8eac-b75cc988b228",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from llama_index import SimpleDirectoryReader\n",
|
|
"\n",
|
|
"from ragas.testset.evolutions import multi_context, reasoning, simple\n",
|
|
"from ragas.testset.generator import TestsetGenerator\n",
|
|
"\n",
|
|
"# load documents\n",
|
|
"dir_path = \"./prompt-engineering-papers\"\n",
|
|
"reader = SimpleDirectoryReader(dir_path, num_files_limit=2)\n",
|
|
"documents = reader.load_data()\n",
|
|
"\n",
|
|
"# generator with openai models\n",
|
|
"generator = TestsetGenerator.with_openai()\n",
|
|
"\n",
|
|
"# set question type distribution\n",
|
|
"distribution = {simple: 0.5, reasoning: 0.25, multi_context: 0.25}\n",
|
|
"\n",
|
|
"# generate testset\n",
|
|
"testset = generator.generate_with_llamaindex_docs(\n",
|
|
" documents, test_size=10, distributions=distribution\n",
|
|
")\n",
|
|
"test_df = testset.to_pandas()\n",
|
|
"test_df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "9c802981-892e-4fed-bb73-dede5540fc6c",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Building RAG"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "72167cb6-bd8a-4d8b-a14c-142235f2ebe0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import nest_asyncio\n",
|
|
"from llama_index import ServiceContext, SimpleDirectoryReader, VectorStoreIndex\n",
|
|
"from llama_index.embeddings import OpenAIEmbedding\n",
|
|
"\n",
|
|
"nest_asyncio.apply()\n",
|
|
"\n",
|
|
"\n",
|
|
"def build_query_engine(documents):\n",
|
|
" vector_index = VectorStoreIndex.from_documents(\n",
|
|
" documents,\n",
|
|
" service_context=ServiceContext.from_defaults(chunk_size=512),\n",
|
|
" embed_model=OpenAIEmbedding(),\n",
|
|
" )\n",
|
|
"\n",
|
|
" query_engine = vector_index.as_query_engine(similarity_top_k=2)\n",
|
|
" return query_engine"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a5e47e5b-fa1a-4f07-b4a4-7493b1d58cc7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"query_engine = build_query_engine(documents)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6469b8ef-f9a3-4fb0-887a-0b70bce59dc0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def generate_single_response(query_engine, question):\n",
|
|
" response = query_engine.query(question)\n",
|
|
" return {\n",
|
|
" \"answer\": response.response,\n",
|
|
" \"contexts\": [c.node.get_content() for c in response.source_nodes],\n",
|
|
" }"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2123caed-a573-4e4e-bb60-41c15de6705f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"question = \"What are some strategies proposed to enhance the in-context learning capability of language models?\"\n",
|
|
"generate_single_response(query_engine, question)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3c88035b-3383-44a6-bd8a-08a172f11a36",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from datasets import Dataset\n",
|
|
"\n",
|
|
"\n",
|
|
"def generate_ragas_dataset(query_engine, test_df):\n",
|
|
" test_questions = test_df[\"question\"].values\n",
|
|
" responses = [generate_single_response(query_engine, q) for q in test_questions]\n",
|
|
"\n",
|
|
" dataset_dict = {\n",
|
|
" \"question\": test_questions,\n",
|
|
" \"answer\": [response[\"answer\"] for response in responses],\n",
|
|
" \"contexts\": [response[\"contexts\"] for response in responses],\n",
|
|
" \"ground_truth\": test_df[\"ground_truth\"].values.tolist(),\n",
|
|
" }\n",
|
|
" ds = Dataset.from_dict(dataset_dict)\n",
|
|
" return ds"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "437368a5-3819-4ae1-b825-ad95664206ae",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"ragas_dataset = generate_ragas_dataset(query_engine, test_df)\n",
|
|
"ragas_df = ragas_dataset.to_pandas()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "10702a1e-276d-45f9-9d81-2be1bd98ce3d",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Commit to Openlayer"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ced5f583-b849-4aae-8397-2bd9006bb69f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import openlayer\n",
|
|
"from openlayer.tasks import TaskType\n",
|
|
"\n",
|
|
"client = openlayer.OpenlayerClient(\"YOUR_OPENLAYER_API_KEY_HERE\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "15c6af02-c9bc-4368-82a1-43cf849446d3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"project = client.create_project(\n",
|
|
" name=\"My-Rag-Project\",\n",
|
|
" task_type=TaskType.LLM,\n",
|
|
" description=\"Evaluating an LLM used for product development.\",\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "419f1392-4c44-4856-af5f-1bd04de1de7c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"validation_dataset_config = {\n",
|
|
" \"contextColumnName\": \"contexts\",\n",
|
|
" \"questionColumnName\": \"question\",\n",
|
|
" \"inputVariableNames\": [\"question\"],\n",
|
|
" \"label\": \"validation\",\n",
|
|
" \"outputColumnName\": \"answer\",\n",
|
|
" \"groundTruthColumnName\": \"ground_truth\",\n",
|
|
"}\n",
|
|
"project.add_dataframe(\n",
|
|
" dataset_df=ragas_df,\n",
|
|
" dataset_config=validation_dataset_config,\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "31c51305-2808-4cae-85c2-b261ca0d98c1",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"model_config = {\n",
|
|
" \"inputVariableNames\": [\"question\"],\n",
|
|
" \"modelType\": \"shell\",\n",
|
|
" \"metadata\": {\"top_k\": 2, \"chunk_size\": 512, \"embeddings\": \"OpenAI\"},\n",
|
|
"}\n",
|
|
"project.add_model(model_config=model_config)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "471643ba-5e5d-4500-9745-f0c355f744a1",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"project.commit(\"Initial commit!\")\n",
|
|
"project.push()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b602dbbc-cc60-48b5-9bab-ae684c61cbff",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.18"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
} |