chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,2 @@
|
||||
OPENAI_API_KEY=your_openai_api_key
|
||||
ANTHROPIC_API_KEY=your_anthropic_api_key
|
||||
@@ -0,0 +1,385 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import opik\n",
|
||||
"opik.configure(use_local=False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from dotenv import load_dotenv\n",
|
||||
"load_dotenv()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import re\n",
|
||||
"import glob\n",
|
||||
"import subprocess\n",
|
||||
"\n",
|
||||
"from IPython.display import Markdown, display\n",
|
||||
"\n",
|
||||
"from llama_index.core import Settings\n",
|
||||
"from llama_index.llms.openai import OpenAI\n",
|
||||
"\n",
|
||||
"from llama_index.core import PromptTemplate\n",
|
||||
"from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
|
||||
"from llama_index.core import VectorStoreIndex, ServiceContext, SimpleDirectoryReader\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"from llama_index.core import Settings\n",
|
||||
"from llama_index.core import PromptTemplate\n",
|
||||
"from llama_index.core import SimpleDirectoryReader\n",
|
||||
"from llama_index.core import VectorStoreIndex\n",
|
||||
"from llama_index.core.storage.storage_context import StorageContext\n",
|
||||
"from llama_index.core.node_parser import CodeSplitter, MarkdownNodeParser\n",
|
||||
"from llama_index.llms.openai import OpenAI\n",
|
||||
"from llama_index.llms.anthropic import Anthropic\n",
|
||||
"from llama_index.core.indices.vector_store.base import VectorStoreIndex\n",
|
||||
"from llama_index.vector_stores.qdrant import QdrantVectorStore\n",
|
||||
"from llama_index.embeddings.fastembed import FastEmbedEmbedding\n",
|
||||
"from llama_index.core import Settings"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Trace RAG calls "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 24,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_index.core import Settings\n",
|
||||
"from llama_index.core.callbacks import CallbackManager\n",
|
||||
"from opik.integrations.llama_index import LlamaIndexCallbackHandler\n",
|
||||
"\n",
|
||||
"# A callback handler tp automatically log all LlamaIndex operations to Opik\n",
|
||||
"opik_callback_handler = LlamaIndexCallbackHandler()\n",
|
||||
"\n",
|
||||
"# Integrate handler into LlamaIndex's settings\n",
|
||||
"Settings.callback_manager = CallbackManager([opik_callback_handler])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"# Step 2: Define helper functions\n",
|
||||
"def parse_github_url(url):\n",
|
||||
" \"\"\"Extract owner and repo name from GitHub URL\"\"\"\n",
|
||||
" pattern = r\"https://github\\.com/([^/]+)/([^/]+)\"\n",
|
||||
" match = re.match(pattern, url)\n",
|
||||
" return match.groups() if match else (None, None)\n",
|
||||
"\n",
|
||||
"def clone_repo(repo_url):\n",
|
||||
" \"\"\"Clone a GitHub repository\"\"\"\n",
|
||||
" return subprocess.run([\"git\", \"clone\", repo_url], check=True, text=True, capture_output=True)\n",
|
||||
"\n",
|
||||
"def parse_docs_by_file_types(ext, language, input_dir_path):\n",
|
||||
" \"\"\"Parse documents based on file extension\"\"\"\n",
|
||||
" files = glob.glob(f\"{input_dir_path}/**/*{ext}\", recursive=True)\n",
|
||||
" \n",
|
||||
" if len(files) > 0:\n",
|
||||
" print(f\"Found {len(files)} files with extension {ext}\")\n",
|
||||
" loader = SimpleDirectoryReader(\n",
|
||||
" input_dir=input_dir_path, required_exts=[ext], recursive=True\n",
|
||||
" )\n",
|
||||
" docs = loader.load_data()\n",
|
||||
" parser = (\n",
|
||||
" MarkdownNodeParser()\n",
|
||||
" if ext == \".md\"\n",
|
||||
" else CodeSplitter.from_defaults(language=language)\n",
|
||||
" )\n",
|
||||
" nodes = parser.get_nodes_from_documents(docs)\n",
|
||||
" print(f\"Processed {len(nodes)} nodes from {ext} files\")\n",
|
||||
" return nodes\n",
|
||||
" return []\n",
|
||||
"\n",
|
||||
"def setup_chat_engine(github_url, model_provider=\"OpenAI o3-mini\"):\n",
|
||||
" \"\"\"\n",
|
||||
" Set up the chat engine for a GitHub repository\n",
|
||||
" Args:\n",
|
||||
" github_url: URL of the GitHub repository\n",
|
||||
" model_provider: 'openai' or 'anthropic'\n",
|
||||
" \"\"\"\n",
|
||||
" # Step 3: Process GitHub URL\n",
|
||||
" owner, repo = parse_github_url(github_url)\n",
|
||||
" if not owner or not repo:\n",
|
||||
" raise ValueError(\"Invalid GitHub URL\")\n",
|
||||
" \n",
|
||||
" print(f\"\\nProcessing repository: {owner}/{repo}\")\n",
|
||||
" input_dir_path = f\"./{repo}\"\n",
|
||||
"\n",
|
||||
" # Step 4: Clone repository if it doesn't exist\n",
|
||||
" if not os.path.exists(input_dir_path):\n",
|
||||
" print(\"\\nCloning repository...\")\n",
|
||||
" clone_repo(github_url)\n",
|
||||
"\n",
|
||||
" # Step 5: Define file types to process\n",
|
||||
" file_types = {\n",
|
||||
" \".md\": \"markdown\",\n",
|
||||
" \".py\": \"python\",\n",
|
||||
" \".ipynb\": \"python\",\n",
|
||||
" \".js\": \"javascript\",\n",
|
||||
" \".ts\": \"typescript\"\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" # Step 6: Process all files\n",
|
||||
" print(\"\\nProcessing files...\")\n",
|
||||
" nodes = []\n",
|
||||
" for ext, language in file_types.items():\n",
|
||||
" nodes += parse_docs_by_file_types(ext, language, input_dir_path)\n",
|
||||
"\n",
|
||||
" if not nodes:\n",
|
||||
" raise ValueError(\"No files were processed from the repository\")\n",
|
||||
"\n",
|
||||
" # Step 7: Setup embedding model\n",
|
||||
" print(\"\\nSetting up embedding model...\")\n",
|
||||
" # Settings.embed_model = FastEmbedEmbedding(model_name=\"BAAI/bge-base-en-v1.5\")\n",
|
||||
"\n",
|
||||
" # Step 8: Create index\n",
|
||||
" print(\"Creating vector index...\")\n",
|
||||
" index = VectorStoreIndex(nodes=nodes)\n",
|
||||
"\n",
|
||||
" # Step 9: Setup LLM and query engine\n",
|
||||
" if model_provider == \"OpenAI o3-mini\":\n",
|
||||
" Settings.llm = OpenAI(model=\"o3-mini\")\n",
|
||||
" elif model_provider == \"Claude 3.7 Sonnet\":\n",
|
||||
" Settings.llm = Anthropic(model=\"claude-3-7-sonnet-20250219\")\n",
|
||||
" elif model_provider == \"Claude 3.5 Sonnet\":\n",
|
||||
" Settings.llm = Anthropic(model=\"claude-3-5-sonnet-20240620\")\n",
|
||||
"\n",
|
||||
" query_engine = index.as_query_engine(streaming=True, similarity_top_k=4)\n",
|
||||
"\n",
|
||||
" # Step 10: Setup custom prompt template\n",
|
||||
" qa_prompt_tmpl_str = (\n",
|
||||
" \"Context information is below.\\n\"\n",
|
||||
" \"---------------------\\n\"\n",
|
||||
" \"{context_str}\\n\"\n",
|
||||
" \"---------------------\\n\"\n",
|
||||
" \"Given the context information above, you must always include a code snippet in your response.\\n\"\n",
|
||||
" \"Think step by step to answer the query, and then provide a relevant code example that demonstrates the concept.\\n\"\n",
|
||||
" \"Even if the question seems conceptual, translate your answer into a practical code example.\\n\"\n",
|
||||
" \"If you don't know the answer, say 'I don't know!' but still provide a minimal code example of what you think might work.\\n\"\n",
|
||||
" \"Query: {query_str}\\n\"\n",
|
||||
" \"Answer: \"\n",
|
||||
" )\n",
|
||||
" qa_prompt_tmpl = PromptTemplate(qa_prompt_tmpl_str)\n",
|
||||
" query_engine.update_prompts(\n",
|
||||
" {\"response_synthesizer:text_qa_template\": qa_prompt_tmpl}\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" print(\"\\nChat engine setup complete! Ready for questions.\")\n",
|
||||
" return query_engine"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model_name = 'Claude 3.7 Sonnet'\n",
|
||||
"github_url = \"https://github.com/Lightning-AI/LitServe\"\n",
|
||||
"query_engine = setup_chat_engine(github_url, model_provider=model_name)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"response = query_engine.query(\"What is this repo about?\") \n",
|
||||
"print(response)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Evaluation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 29,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from opik import Opik\n",
|
||||
"\n",
|
||||
"client = Opik()\n",
|
||||
"dataset = client.get_or_create_dataset(name=\"Eval Code Generation\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 30,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from opik import track\n",
|
||||
"\n",
|
||||
"@track\n",
|
||||
"def my_llm_application(input: str) -> str:\n",
|
||||
" response = query_engine.query(input)\n",
|
||||
" return str(response)\n",
|
||||
"\n",
|
||||
"def evaluation_task(x):\n",
|
||||
" return {\n",
|
||||
" \"output\": my_llm_application(x['input'])\n",
|
||||
" }"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 32,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from opik.evaluation.metrics import base_metric, score_result\n",
|
||||
"from openai import OpenAI\n",
|
||||
"from typing import Any\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"class LLMJudgeMetric(base_metric.BaseMetric):\n",
|
||||
" def __init__(self, name: str = \"Code Quality Evaluation\", model_name: str = \"gpt-4o\"):\n",
|
||||
" self.name = name\n",
|
||||
" self.llm_client = OpenAI()\n",
|
||||
" self.model_name = model_name\n",
|
||||
" self.prompt_template = \"\"\"\n",
|
||||
" You are an expert judge tasked with evaluating the quality of code generation by comparing the AI-generated code to the ground truth code.\n",
|
||||
" \n",
|
||||
" Evaluate how well the AI-generated code matches the ground truth code in terms of:\n",
|
||||
" 1. Correctness: Does the generated code implement the same functionality?\n",
|
||||
" 2. Completeness: Does the generated code include all necessary components?\n",
|
||||
" 3. Efficiency: Is the generated code similarly efficient in its approach?\n",
|
||||
" 4. If the generated code is not exactly the same as the ground truth, but the functionality is similar, then still give a high score.\n",
|
||||
" 5. Only focus on the code and the functionality, ignore the text.\n",
|
||||
" \n",
|
||||
" The format of your response should be a JSON object with no additional text or backticks that follows the format:\n",
|
||||
" {{\n",
|
||||
" \"score\": <score between 0 and 1>\n",
|
||||
" }}\n",
|
||||
" \n",
|
||||
" Where:\n",
|
||||
" - 0 means the generated code is completely different or incorrect\n",
|
||||
" - 1 means the generated code is functionally equivalent to the ground truth\n",
|
||||
" \n",
|
||||
" AI-generated code: {output}\n",
|
||||
" \n",
|
||||
" Response:\n",
|
||||
" \"\"\"\n",
|
||||
" def score(self, output: str, **ignored_kwargs: Any):\n",
|
||||
" \"\"\"\n",
|
||||
" Score the output of an LLM.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" output: The output of an LLM to score.\n",
|
||||
" **ignored_kwargs: Any additional keyword arguments. This is important so that the metric can be used in the `evaluate` function.\n",
|
||||
" \"\"\"\n",
|
||||
" # Construct the prompt based on the output of the LLM\n",
|
||||
" prompt = self.prompt_template.format(output=output)\n",
|
||||
" # Generate and parse the response from the LLM\n",
|
||||
" response = self.llm_client.chat.completions.create(\n",
|
||||
" model=self.model_name,\n",
|
||||
" messages=[{\"role\": \"user\", \"content\": prompt}]\n",
|
||||
" )\n",
|
||||
" response_dict = json.loads(response.choices[0].message.content)\n",
|
||||
"\n",
|
||||
" response_score = float(response_dict[\"score\"])\n",
|
||||
"\n",
|
||||
" return score_result.ScoreResult(\n",
|
||||
" name=self.name,\n",
|
||||
" value=response_score\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 33,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"code_quality_metric = LLMJudgeMetric()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from opik.evaluation import evaluate\n",
|
||||
"\n",
|
||||
"evaluation = evaluate(\n",
|
||||
" dataset=dataset,\n",
|
||||
" task=evaluation_task,\n",
|
||||
" experiment_name = model_name,\n",
|
||||
" scoring_metrics=[code_quality_metric],\n",
|
||||
" experiment_config={\n",
|
||||
" \"model\": \"gpt-3.5-turbo\"\n",
|
||||
" }\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "env_gen",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.11"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
|
||||
# Compare Claud 3.7 Sonnet and OpenAI o3 using RAG over code (GitHub).
|
||||
|
||||
This project will also leverages [CometML Opik](https://github.com/comet-ml/opik) to build an e2e evaluation and observability pipeline for a RAG application.
|
||||
|
||||
|
||||
## Installation and setup
|
||||
|
||||
**Get API Keys**:
|
||||
- [Opik API Key](https://www.comet.com/signup)
|
||||
- [Open AI API Key](https://platform.openai.com/api-keys)
|
||||
- [Anthropic AI API Key](https://www.anthropic.com/api)
|
||||
|
||||
Add these to your .env file, refer ```.env.example```
|
||||
|
||||
|
||||
|
||||
**Install Dependencies**:
|
||||
Ensure you have Python 3.11 or later installed.
|
||||
```bash
|
||||
pip install opik llama-index llama-index-agent-openai llama-index-llms-openai llama-index-llms-anthropic --upgrade --quiet
|
||||
```
|
||||
|
||||
**Running the app**:
|
||||
|
||||
Run streamlit app using ``` streamlit run app.py```.
|
||||
|
||||
**Running Evaluation**:
|
||||
|
||||
You can run the code in notebook ```Opik for LLM evaluation.ipynb```.
|
||||
|
||||
---
|
||||
|
||||
## 📬 Stay Updated with Our Newsletter!
|
||||
**Get a FREE Data Science eBook** 📖 with 150+ essential lessons in Data Science when you subscribe to our newsletter! Stay in the loop with the latest tutorials, insights, and exclusive resources. [Subscribe now!](https://join.dailydoseofds.com)
|
||||
|
||||
[](https://join.dailydoseofds.com)
|
||||
|
||||
---
|
||||
|
||||
## Contribution
|
||||
|
||||
Contributions are welcome! Please fork the repository and submit a pull request with your improvements.
|
||||
@@ -0,0 +1,243 @@
|
||||
import os
|
||||
|
||||
import gc
|
||||
import re
|
||||
import glob
|
||||
import uuid
|
||||
import textwrap
|
||||
import subprocess
|
||||
import nest_asyncio
|
||||
from dotenv import load_dotenv
|
||||
|
||||
import streamlit as st
|
||||
|
||||
from llama_index.core import Settings
|
||||
from llama_index.core import PromptTemplate
|
||||
from llama_index.core import SimpleDirectoryReader
|
||||
from llama_index.core import VectorStoreIndex
|
||||
from llama_index.core.storage.storage_context import StorageContext
|
||||
from llama_index.core.node_parser import CodeSplitter, MarkdownNodeParser
|
||||
from llama_index.llms.openai import OpenAI
|
||||
from llama_index.llms.anthropic import Anthropic
|
||||
from llama_index.core.indices.vector_store.base import VectorStoreIndex
|
||||
from llama_index.vector_stores.qdrant import QdrantVectorStore
|
||||
from llama_index.embeddings.fastembed import FastEmbedEmbedding
|
||||
import qdrant_client
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# setting up the llm
|
||||
@st.cache_resource
|
||||
def load_llm(model_name, provider="openai"):
|
||||
if provider == "anthropic":
|
||||
return Anthropic(model=model_name)
|
||||
elif provider == "openai":
|
||||
return OpenAI(model=model_name)
|
||||
else:
|
||||
raise ValueError(f"Unsupported provider: {provider}")
|
||||
|
||||
|
||||
# utility functions
|
||||
def parse_github_url(url):
|
||||
pattern = r"https://github\.com/([^/]+)/([^/]+)"
|
||||
match = re.match(pattern, url)
|
||||
return match.groups() if match else (None, None)
|
||||
|
||||
def clone_repo(repo_url):
|
||||
return subprocess.run(["git", "clone", repo_url], check=True, text=True, capture_output=True)
|
||||
|
||||
|
||||
def validate_owner_repo(owner, repo):
|
||||
return bool(owner) and bool(repo)
|
||||
|
||||
def parse_docs_by_file_types(ext, language, input_dir_path):
|
||||
"""Parse documents based on file extension"""
|
||||
files = glob.glob(f"{input_dir_path}/**/*{ext}", recursive=True)
|
||||
|
||||
if len(files) > 0:
|
||||
print(f"Found {len(files)} files with extension {ext}")
|
||||
loader = SimpleDirectoryReader(
|
||||
input_dir=input_dir_path, required_exts=[ext], recursive=True
|
||||
)
|
||||
docs = loader.load_data()
|
||||
parser = (
|
||||
MarkdownNodeParser()
|
||||
if ext == ".md"
|
||||
else CodeSplitter.from_defaults(language=language)
|
||||
)
|
||||
nodes = parser.get_nodes_from_documents(docs)
|
||||
print(f"Processed {len(nodes)} nodes from {ext} files")
|
||||
return nodes
|
||||
return []
|
||||
|
||||
|
||||
# create an qdrant collection and return an index
|
||||
def create_index(nodes, client):
|
||||
unique_collection_id = uuid.uuid4()
|
||||
collection_name = f"chat_with_docs_{unique_collection_id}"
|
||||
vector_store = QdrantVectorStore(client=client, collection_name=collection_name)
|
||||
storage_context = StorageContext.from_defaults(vector_store=vector_store)
|
||||
index = VectorStoreIndex(
|
||||
nodes,
|
||||
storage_context=storage_context,
|
||||
)
|
||||
return index
|
||||
|
||||
if "id" not in st.session_state:
|
||||
st.session_state.id = uuid.uuid4()
|
||||
st.session_state.file_cache = {}
|
||||
|
||||
session_id = st.session_state.id
|
||||
client = None
|
||||
|
||||
|
||||
def reset_chat():
|
||||
st.session_state.messages = []
|
||||
st.session_state.context = None
|
||||
gc.collect()
|
||||
|
||||
|
||||
with st.sidebar:
|
||||
|
||||
# Model selection
|
||||
model_options = {
|
||||
"OpenAI o3-mini": {"provider": "openai", "model": "o3-mini"},
|
||||
"Claude 3.7 Sonnet": {"provider": "anthropic", "model": "claude-3-7-sonnet-20250219"}
|
||||
}
|
||||
selected_model = st.selectbox(
|
||||
"Select Model",
|
||||
options=list(model_options.keys()),
|
||||
index=0
|
||||
)
|
||||
|
||||
# Input for GitHub URL
|
||||
github_url = st.text_input("GitHub Repository URL")
|
||||
|
||||
# Button to load and process the GitHub repository
|
||||
process_button = st.button("Load")
|
||||
|
||||
message_container = st.empty() # Placeholder for dynamic messages
|
||||
|
||||
if process_button and github_url:
|
||||
owner, repo = parse_github_url(github_url)
|
||||
if validate_owner_repo(owner, repo):
|
||||
with st.spinner(f"Loading {repo} repository by {owner}..."):
|
||||
try:
|
||||
input_dir_path = f"./{repo}"
|
||||
|
||||
if not os.path.exists(input_dir_path):
|
||||
subprocess.run(["git", "clone", github_url], check=True, text=True, capture_output=True)
|
||||
|
||||
if os.path.exists(input_dir_path):
|
||||
file_types = {
|
||||
".md": "markdown",
|
||||
".py": "python",
|
||||
".ipynb": "python",
|
||||
".js": "javascript",
|
||||
".ts": "typescript"
|
||||
}
|
||||
|
||||
nodes = []
|
||||
for ext, language in file_types.items():
|
||||
nodes += parse_docs_by_file_types(ext, language, input_dir_path)
|
||||
else:
|
||||
st.error('Error occurred while cloning the repository, carefully check the url')
|
||||
st.stop()
|
||||
|
||||
# setting up the embedding model
|
||||
Settings.embed_model = FastEmbedEmbedding(model_name="BAAI/bge-base-en-v1.5")
|
||||
try:
|
||||
index = create_index(nodes)
|
||||
except:
|
||||
index = VectorStoreIndex(nodes=nodes)
|
||||
|
||||
# ====== Setup a query engine ======
|
||||
model_info = model_options[selected_model]
|
||||
Settings.llm = load_llm(model_name=model_info["model"], provider=model_info["provider"])
|
||||
query_engine = index.as_query_engine(streaming=True, similarity_top_k=4)
|
||||
|
||||
# ====== Customise prompt template ======
|
||||
qa_prompt_tmpl_str = (
|
||||
"Context information is below.\n"
|
||||
"---------------------\n"
|
||||
"{context_str}\n"
|
||||
"---------------------\n"
|
||||
"Given the context information and your knowledge, I want you to think step by step to answer the query in a crisp manner, incase case you don't know the answer say 'I don't know!'.\n"
|
||||
"Query: {query_str}\n"
|
||||
"Answer: "
|
||||
)
|
||||
qa_prompt_tmpl = PromptTemplate(qa_prompt_tmpl_str)
|
||||
|
||||
query_engine.update_prompts(
|
||||
{"response_synthesizer:text_qa_template": qa_prompt_tmpl}
|
||||
)
|
||||
|
||||
if nodes:
|
||||
message_container.success("Data loaded successfully!!")
|
||||
else:
|
||||
message_container.write(
|
||||
"No data found, check if the repository is not empty!"
|
||||
)
|
||||
st.session_state.query_engine = query_engine
|
||||
|
||||
except Exception as e:
|
||||
st.error(f"An error occurred: {e}")
|
||||
st.stop()
|
||||
|
||||
st.success("Ready to Chat!")
|
||||
else:
|
||||
st.error('Invalid owner or repository')
|
||||
st.stop()
|
||||
|
||||
col1, col2 = st.columns([6, 1])
|
||||
|
||||
with col1:
|
||||
st.header(f"Claude 3.7 Sonnet vs OpenAI o3! </>")
|
||||
|
||||
with col2:
|
||||
st.button("Clear ↺", on_click=reset_chat)
|
||||
|
||||
|
||||
# Initialize chat history
|
||||
if "messages" not in st.session_state:
|
||||
reset_chat()
|
||||
|
||||
|
||||
# Display chat messages from history on app rerun
|
||||
for message in st.session_state.messages:
|
||||
with st.chat_message(message["role"]):
|
||||
st.markdown(message["content"])
|
||||
|
||||
|
||||
# Accept user input
|
||||
if prompt := st.chat_input("What's up?"):
|
||||
# Add user message to chat history
|
||||
st.session_state.messages.append({"role": "user", "content": prompt})
|
||||
# Display user message in chat message container
|
||||
with st.chat_message("user"):
|
||||
st.markdown(prompt)
|
||||
|
||||
# Display assistant response in chat message container
|
||||
with st.chat_message("assistant"):
|
||||
message_placeholder = st.empty()
|
||||
full_response = ""
|
||||
|
||||
# context = st.session_state.context
|
||||
query_engine = st.session_state.query_engine
|
||||
|
||||
# Simulate stream of response with milliseconds delay
|
||||
streaming_response = query_engine.query(prompt)
|
||||
|
||||
for chunk in streaming_response.response_gen:
|
||||
full_response += chunk
|
||||
message_placeholder.markdown(full_response + "▌")
|
||||
|
||||
# full_response = query_engine.query(prompt)
|
||||
|
||||
message_placeholder.markdown(full_response)
|
||||
# st.session_state.context = ctx
|
||||
|
||||
# Add assistant response to chat history
|
||||
st.session_state.messages.append({"role": "assistant", "content": full_response})
|
||||
@@ -0,0 +1,133 @@
|
||||
question,answer
|
||||
"Write a SimpleLitAPI that takes a number calculates it's square, calculates it cube and returns the response.","# server.py
|
||||
import litserve as ls
|
||||
|
||||
# (STEP 1) - DEFINE THE API (compound AI system)
|
||||
class SimpleLitAPI(ls.LitAPI):
|
||||
def setup(self, device):
|
||||
# setup is called once at startup. Build a compound AI system (1+ models), connect DBs, load data, etc...
|
||||
self.model1 = lambda x: x**2
|
||||
self.model2 = lambda x: x**3
|
||||
|
||||
def decode_request(self, request):
|
||||
# Convert the request payload to model input.
|
||||
return request[""input""]
|
||||
|
||||
def predict(self, x):
|
||||
# Easily build compound systems. Run inference and return the output.
|
||||
squared = self.model1(x)
|
||||
cubed = self.model2(x)
|
||||
output = squared + cubed
|
||||
return {""output"": output}
|
||||
|
||||
def encode_response(self, output):
|
||||
# Convert the model output to a response payload.
|
||||
return {""output"": output}
|
||||
|
||||
# (STEP 2) - START THE SERVER
|
||||
if __name__ == ""__main__"":
|
||||
# scale with advanced features (batching, GPUs, etc...)
|
||||
server = ls.LitServer(SimpleLitAPI(), accelerator=""auto"", max_batch_size=1)
|
||||
server.run(port=8000)"
|
||||
"Build a text embedding API using SentenceTransformer and Litserve.","from sentence_transformers import SentenceTransformer
|
||||
import litserve as ls
|
||||
|
||||
class EmbeddingAPI(ls.LitAPI):
|
||||
def setup(self, device):
|
||||
self.instruction = ""Represent this sentence for searching relevant passages: ""
|
||||
self.model = SentenceTransformer('BAAI/bge-large-en-v1.5', device=device)
|
||||
|
||||
def decode_request(self, request):
|
||||
return request[""input""]
|
||||
|
||||
def predict(self, query):
|
||||
return self.model.encode([self.instruction + query], normalize_embeddings=True)
|
||||
|
||||
def encode_response(self, output):
|
||||
return {""embedding"": output[0].tolist()}
|
||||
|
||||
if __name__ == ""__main__"":
|
||||
api = EmbeddingAPI()
|
||||
server = ls.LitServer(api)
|
||||
server.run(port=8000)"
|
||||
"Create a LitServe RAG API using LlamaIndex, Qdrant as vector database and Ollama to serve llama3.2 locally","import os, logging, qdrant_client
|
||||
from llama_index.llms.ollama import Ollama
|
||||
from llama_index.core import StorageContext, Settings, VectorStoreIndex, SimpleDirectoryReader
|
||||
from llama_index.vector_stores.qdrant import QdrantVectorStore
|
||||
from llama_index.embeddings.fastembed import FastEmbedEmbedding
|
||||
import litserve as ls
|
||||
|
||||
class DocumentChatAPI(ls.LitAPI):
|
||||
def setup(self, device):
|
||||
Settings.llm = Ollama(model=""llama3.1:latest"", request_timeout=120.0)
|
||||
Settings.embed_model = FastEmbedEmbedding(model_name=""BAAI/bge-large-en-v1.5"")
|
||||
client = qdrant_client.QdrantClient(host=""localhost"", port=6333)
|
||||
vector_store = QdrantVectorStore(client=client, collection_name=""doc_search_collection"")
|
||||
storage_context = StorageContext.from_defaults(vector_store=vector_store)
|
||||
documents = SimpleDirectoryReader(""./docs"").load_data()
|
||||
index = VectorStoreIndex.from_documents(documents, storage_context=storage_context)
|
||||
self.query_engine = index.as_query_engine()
|
||||
|
||||
def decode_request(self, request):
|
||||
return request[""query""]
|
||||
|
||||
def predict(self, query):
|
||||
return self.query_engine.query(query)
|
||||
|
||||
def encode_response(self, output):
|
||||
return {""output"": output}
|
||||
|
||||
if __name__ == ""__main__"":
|
||||
api = DocumentChatAPI()
|
||||
server = ls.LitServer(api)
|
||||
server.run(port=8000)"
|
||||
"Create a private API for Open AI's Whisper model using LitServe","# whisper_server.py
|
||||
import litserve as ls
|
||||
import whisper
|
||||
|
||||
class WhisperLitAPI(ls.LitAPI):
|
||||
def setup(self, device):
|
||||
# Load the OpenAI Whisper model. You can specify other models like ""base"", ""small"", etc.
|
||||
self.model = whisper.load_model(""large"", device='cuda')
|
||||
|
||||
def decode_request(self, request):
|
||||
# Assuming the request sends the path to the audio file
|
||||
# In a more robust implementation, you would handle audio data directly.
|
||||
return request[""audio_path""]
|
||||
|
||||
def predict(self, audio_path):
|
||||
# Process the audio file and return the transcription result
|
||||
result = self.model.transcribe(audio_path)
|
||||
return result
|
||||
|
||||
def encode_response(self, output):
|
||||
# Return the transcription text
|
||||
return {""transcription"": output[""text""]}
|
||||
|
||||
if __name__ == ""__main__"":
|
||||
api = WhisperLitAPI()
|
||||
server = ls.LitServer(api, accelerator=""gpu"", timeout=1000, workers_per_device=2)
|
||||
server.run(port=8000)"
|
||||
"Deploy a random forest model using LitServe","import pickle, numpy as np
|
||||
import litserve as ls
|
||||
|
||||
class RandomForestAPI(ls.LitAPI):
|
||||
def setup(self, device):
|
||||
with open(""model.pkl"", ""rb"") as f:
|
||||
self.model = pickle.load(f)
|
||||
|
||||
def decode_request(self, request):
|
||||
x = np.asarray(request[""input""])
|
||||
x = np.expand_dims(x, 0)
|
||||
return x
|
||||
|
||||
def predict(self, x):
|
||||
return self.model.predict(x)
|
||||
|
||||
def encode_response(self, output):
|
||||
return {""class_idx"": int(output)}
|
||||
|
||||
if __name__ == ""__main__"":
|
||||
api = RandomForestAPI()
|
||||
server = ls.LitServer(api)
|
||||
server.run(port=8000)"
|
||||
|
Reference in New Issue
Block a user