chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,52 @@
|
||||
# LLama3.3-RAG application
|
||||
|
||||
This project build the fastest stack to build a RAG application to **chat with your docs**.
|
||||
We use:
|
||||
- SambaNova as the inference engine for Llama 3.3.
|
||||
- Llama index for orchestrating the RAG app.
|
||||
- Qdrant VectorDB for storing the embeddings.
|
||||
- Streamlit to build the UI.
|
||||
|
||||
## Installation and setup
|
||||
|
||||
**Setup SambaNova**:
|
||||
|
||||
Get an API key from [SambaNova](https://sambanova.ai/) and set it in the `.env` file as follows:
|
||||
|
||||
```bash
|
||||
SAMBANOVA_API_KEY=<YOUR_SAMBANOVA_API_KEY>
|
||||
```
|
||||
|
||||
**Setup Qdrant VectorDB**
|
||||
```bash
|
||||
docker run -p 6333:6333 -p 6334:6334 \
|
||||
-v $(pwd)/qdrant_storage:/qdrant/storage:z \
|
||||
qdrant/qdrant
|
||||
```
|
||||
|
||||
**Install Dependencies**:
|
||||
Ensure you have Python 3.11 or later installed.
|
||||
```bash
|
||||
pip install streamlit llama-index-vector-stores-qdrant llama-index-llms-sambanovasystems sseclient-py
|
||||
```
|
||||
|
||||
**Run the app**:
|
||||
|
||||
Run the app by running the following command:
|
||||
|
||||
```bash
|
||||
streamlit run app.py
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📬 Stay Updated with Our Newsletter!
|
||||
**Get a FREE Data Science eBook** 📖 with 150+ essential lessons in Data Science when you subscribe to our newsletter! Stay in the loop with the latest tutorials, insights, and exclusive resources. [Subscribe now!](https://join.dailydoseofds.com)
|
||||
|
||||
[](https://join.dailydoseofds.com)
|
||||
|
||||
---
|
||||
|
||||
## Contribution
|
||||
|
||||
Contributions are welcome! Please fork the repository and submit a pull request with your improvements.
|
||||
@@ -0,0 +1,157 @@
|
||||
# Adapted from https://docs.streamlit.io/knowledge-base/tutorials/build-conversational-apps#build-a-simple-chatbot-gui-with-streaming
|
||||
import os
|
||||
import openai
|
||||
import base64
|
||||
import gc
|
||||
import random
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from IPython.display import Markdown, display
|
||||
from dotenv import load_dotenv
|
||||
from llama_index.core import SimpleDirectoryReader
|
||||
from rag_code import EmbedData, QdrantVDB_QB, Retriever, RAG
|
||||
|
||||
import streamlit as st
|
||||
|
||||
if "id" not in st.session_state:
|
||||
st.session_state.id = uuid.uuid4()
|
||||
st.session_state.file_cache = {}
|
||||
|
||||
session_id = st.session_state.id
|
||||
collection_name = "chat with docs"
|
||||
batch_size = 32
|
||||
|
||||
load_dotenv()
|
||||
|
||||
def reset_chat():
|
||||
st.session_state.messages = []
|
||||
st.session_state.context = None
|
||||
gc.collect()
|
||||
|
||||
|
||||
def display_pdf(file):
|
||||
# Opening file from file path
|
||||
|
||||
st.markdown("### PDF Preview")
|
||||
base64_pdf = base64.b64encode(file.read()).decode("utf-8")
|
||||
|
||||
# Embedding PDF in HTML
|
||||
pdf_display = f"""<iframe src="data:application/pdf;base64,{base64_pdf}" width="400" height="100%" type="application/pdf"
|
||||
style="height:100vh; width:100%"
|
||||
>
|
||||
</iframe>"""
|
||||
|
||||
# Displaying File
|
||||
st.markdown(pdf_display, unsafe_allow_html=True)
|
||||
|
||||
|
||||
with st.sidebar:
|
||||
st.header(f"Add your documents!")
|
||||
|
||||
uploaded_file = st.file_uploader("Choose your `.pdf` file", type="pdf")
|
||||
|
||||
if uploaded_file:
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
file_path = os.path.join(temp_dir, uploaded_file.name)
|
||||
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(uploaded_file.getvalue())
|
||||
|
||||
file_key = f"{session_id}-{uploaded_file.name}"
|
||||
st.write("Indexing your document...")
|
||||
|
||||
if file_key not in st.session_state.get('file_cache', {}):
|
||||
|
||||
if os.path.exists(temp_dir):
|
||||
loader = SimpleDirectoryReader(
|
||||
input_dir = temp_dir,
|
||||
required_exts=[".pdf"],
|
||||
recursive=True
|
||||
)
|
||||
else:
|
||||
st.error('Could not find the file you uploaded, please check again...')
|
||||
st.stop()
|
||||
|
||||
docs = loader.load_data()
|
||||
documents = [doc.text for doc in docs]
|
||||
|
||||
# embed data
|
||||
embeddata = EmbedData(embed_model_name="BAAI/bge-large-en-v1.5", batch_size=batch_size)
|
||||
embeddata.embed(documents)
|
||||
|
||||
# set up vector database
|
||||
qdrant_vdb = QdrantVDB_QB(collection_name=collection_name,
|
||||
batch_size=batch_size,
|
||||
vector_dim=1024)
|
||||
qdrant_vdb.define_client()
|
||||
qdrant_vdb.create_collection()
|
||||
qdrant_vdb.ingest_data(embeddata=embeddata)
|
||||
|
||||
# set up retriever
|
||||
retriever = Retriever(vector_db=qdrant_vdb, embeddata=embeddata)
|
||||
|
||||
# set up rag
|
||||
query_engine = RAG(retriever=retriever, llm_name="Meta-Llama-3.3-70B-Instruct")
|
||||
|
||||
st.session_state.file_cache[file_key] = query_engine
|
||||
|
||||
else:
|
||||
query_engine = st.session_state.file_cache[file_key]
|
||||
|
||||
# Inform the user that the file is processed and Display the PDF uploaded
|
||||
st.success("Ready to Chat!")
|
||||
display_pdf(uploaded_file)
|
||||
except Exception as e:
|
||||
st.error(f"An error occurred: {e}")
|
||||
st.stop()
|
||||
|
||||
col1, col2 = st.columns([6, 1])
|
||||
|
||||
with col1:
|
||||
st.header(f"Fastest RAG Stack with SambaNova and Llama-3.3")
|
||||
|
||||
with col2:
|
||||
st.button("Clear ↺", on_click=reset_chat)
|
||||
|
||||
# Initialize chat history
|
||||
if "messages" not in st.session_state:
|
||||
reset_chat()
|
||||
|
||||
|
||||
# Display chat messages from history on app rerun
|
||||
for message in st.session_state.messages:
|
||||
with st.chat_message(message["role"]):
|
||||
st.markdown(message["content"])
|
||||
|
||||
|
||||
# Accept user input
|
||||
if prompt := st.chat_input("What's up?"):
|
||||
# Add user message to chat history
|
||||
st.session_state.messages.append({"role": "user", "content": prompt})
|
||||
# Display user message in chat message container
|
||||
with st.chat_message("user"):
|
||||
st.markdown(prompt)
|
||||
|
||||
# Display assistant response in chat message container
|
||||
with st.chat_message("assistant"):
|
||||
message_placeholder = st.empty()
|
||||
full_response = ""
|
||||
|
||||
# Simulate stream of response with milliseconds delay
|
||||
streaming_response = query_engine.query(prompt)
|
||||
|
||||
for chunk in streaming_response:
|
||||
try:
|
||||
new_text = chunk.raw["choices"][0]["delta"]["content"]
|
||||
full_response += new_text
|
||||
message_placeholder.markdown(full_response + "▌")
|
||||
except:
|
||||
pass
|
||||
|
||||
message_placeholder.markdown(full_response)
|
||||
|
||||
# Add assistant response to chat history
|
||||
st.session_state.messages.append({"role": "assistant", "content": full_response})
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
@@ -0,0 +1,168 @@
|
||||
from qdrant_client import models
|
||||
from qdrant_client import QdrantClient
|
||||
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
|
||||
from llama_index.llms.sambanovasystems import SambaNovaCloud
|
||||
|
||||
from llama_index.core.base.llms.types import (
|
||||
ChatMessage,
|
||||
MessageRole,
|
||||
)
|
||||
|
||||
def batch_iterate(lst, batch_size):
|
||||
"""Yield successive n-sized chunks from lst."""
|
||||
for i in range(0, len(lst), batch_size):
|
||||
yield lst[i : i + batch_size]
|
||||
|
||||
class EmbedData:
|
||||
|
||||
def __init__(self, embed_model_name="BAAI/bge-large-en-v1.5", batch_size = 32):
|
||||
self.embed_model_name = embed_model_name
|
||||
self.embed_model = self._load_embed_model()
|
||||
self.batch_size = batch_size
|
||||
self.embeddings = []
|
||||
|
||||
def _load_embed_model(self):
|
||||
embed_model = HuggingFaceEmbedding(model_name=self.embed_model_name, trust_remote_code=True, cache_folder='./hf_cache')
|
||||
return embed_model
|
||||
|
||||
def generate_embedding(self, context):
|
||||
return self.embed_model.get_text_embedding_batch(context)
|
||||
|
||||
def embed(self, contexts):
|
||||
|
||||
self.contexts = contexts
|
||||
|
||||
for batch_context in batch_iterate(contexts, self.batch_size):
|
||||
batch_embeddings = self.generate_embedding(batch_context)
|
||||
self.embeddings.extend(batch_embeddings)
|
||||
|
||||
class QdrantVDB_QB:
|
||||
|
||||
def __init__(self, collection_name, vector_dim = 768, batch_size=512):
|
||||
self.collection_name = collection_name
|
||||
self.batch_size = batch_size
|
||||
self.vector_dim = vector_dim
|
||||
|
||||
def define_client(self):
|
||||
|
||||
self.client = QdrantClient(url="http://localhost:6333", prefer_grpc=True)
|
||||
|
||||
def create_collection(self):
|
||||
|
||||
if not self.client.collection_exists(collection_name=self.collection_name):
|
||||
|
||||
self.client.create_collection(collection_name=f"{self.collection_name}",
|
||||
|
||||
vectors_config=models.VectorParams(size=self.vector_dim,
|
||||
distance=models.Distance.DOT,
|
||||
on_disk=True),
|
||||
|
||||
optimizers_config=models.OptimizersConfigDiff(default_segment_number=5,
|
||||
indexing_threshold=0),
|
||||
|
||||
quantization_config=models.BinaryQuantization(
|
||||
binary=models.BinaryQuantizationConfig(always_ram=True)),
|
||||
)
|
||||
|
||||
def ingest_data(self, embeddata):
|
||||
|
||||
for batch_context, batch_embeddings in zip(batch_iterate(embeddata.contexts, self.batch_size),
|
||||
batch_iterate(embeddata.embeddings, self.batch_size)):
|
||||
|
||||
self.client.upload_collection(collection_name=self.collection_name,
|
||||
vectors=batch_embeddings,
|
||||
payload=[{"context": context} for context in batch_context])
|
||||
|
||||
self.client.update_collection(collection_name=self.collection_name,
|
||||
optimizer_config=models.OptimizersConfigDiff(indexing_threshold=20000)
|
||||
)
|
||||
|
||||
class Retriever:
|
||||
|
||||
def __init__(self, vector_db, embeddata):
|
||||
|
||||
self.vector_db = vector_db
|
||||
self.embeddata = embeddata
|
||||
|
||||
def search(self, query):
|
||||
query_embedding = self.embeddata.embed_model.get_query_embedding(query)
|
||||
|
||||
|
||||
result = self.vector_db.client.search(
|
||||
collection_name=self.vector_db.collection_name,
|
||||
|
||||
query_vector=query_embedding,
|
||||
|
||||
search_params=models.SearchParams(
|
||||
quantization=models.QuantizationSearchParams(
|
||||
ignore=False,
|
||||
rescore=True,
|
||||
oversampling=2.0,
|
||||
)
|
||||
),
|
||||
|
||||
timeout=1000,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
class RAG:
|
||||
|
||||
def __init__(self,
|
||||
retriever,
|
||||
llm_name = "llama3.2:1b"
|
||||
):
|
||||
|
||||
system_msg = ChatMessage(
|
||||
role=MessageRole.SYSTEM,
|
||||
content="You are a helpful assistant that answers questions about the user's document.",
|
||||
)
|
||||
self.messages = [system_msg, ]
|
||||
self.llm_name = llm_name
|
||||
self.llm = self._setup_llm()
|
||||
self.retriever = retriever
|
||||
self.qa_prompt_tmpl_str = ("Context information is below.\n"
|
||||
"---------------------\n"
|
||||
"{context}\n"
|
||||
"---------------------\n"
|
||||
"Given the context information above I want you to think step by step to answer the query in a crisp manner, incase case you don't know the answer say 'I don't know!'.\n"
|
||||
"Query: {query}\n"
|
||||
"Answer: "
|
||||
)
|
||||
|
||||
def _setup_llm(self):
|
||||
return SambaNovaCloud(
|
||||
model=self.llm_name,
|
||||
temperature=0.7,
|
||||
context_window=100000,
|
||||
)
|
||||
|
||||
def generate_context(self, query):
|
||||
|
||||
result = self.retriever.search(query)
|
||||
context = [dict(data) for data in result]
|
||||
combined_prompt = []
|
||||
|
||||
for entry in context[:2]:
|
||||
context = entry["payload"]["context"]
|
||||
|
||||
combined_prompt.append(context)
|
||||
|
||||
return "\n\n---\n\n".join(combined_prompt)
|
||||
|
||||
def query(self, query):
|
||||
context = self.generate_context(query=query)
|
||||
|
||||
prompt = self.qa_prompt_tmpl_str.format(context=context, query=query)
|
||||
|
||||
user_msg = ChatMessage(role=MessageRole.USER, content=prompt)
|
||||
|
||||
# self.messages.append(ChatMessage(role=MessageRole.USER, content=prompt))
|
||||
|
||||
streaming_response = self.llm.stream_complete(user_msg.content)
|
||||
|
||||
return streaming_response
|
||||
|
||||
# def append_ai_response(self, message):
|
||||
|
||||
# self.messages.append(ChatMessage(role=MessageRole.ASSISTANT, content=message))
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 284 KiB |
@@ -0,0 +1,129 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from dotenv import load_dotenv\n",
|
||||
"from llama_index.llms.sambanovasystems import SambaNovaCloud\n",
|
||||
"\n",
|
||||
"load_dotenv()\n",
|
||||
"\n",
|
||||
"MODEL = \"Meta-Llama-3.1-405B-Instruct\"\n",
|
||||
"\n",
|
||||
"llm = SambaNovaCloud(model=MODEL)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"The history of Artificial Intelligence (AI) spans several decades, and it is a story of continuous innovation, experimentation, and advancement. Here's a comprehensive overview of the major milestones in the development of AI:\n",
|
||||
"\n",
|
||||
"**Early Beginnings (1950s)**\n",
|
||||
"\n",
|
||||
"The term \"Artificial Intelligence\" was first coined in 1956 by John McCarthy, a computer scientist and cognitive scientist, at the Dartmouth Summer Research Project on Artificial Intelligence. This conference is considered the birthplace of AI as a field of research.\n",
|
||||
"\n",
|
||||
"In the 1950s, computer scientists like Alan Turing, Marvin Minsky, and Seymour Papert began exploring the idea of creating machines that could think and learn like humans. Turing's 1950 paper, \"Computing Machinery and Intelligence,\" proposed the Turing Test, a measure of a machine's ability to exhibit intelligent behavior equivalent to, or indistinguishable from, that of a human.\n",
|
||||
"\n",
|
||||
"**Rule-Based Expert Systems (1960s-1970s)**\n",
|
||||
"\n",
|
||||
"In the 1960s and 1970s, AI research focused on developing rule-based expert systems. These systems mimicked human decision-making by using a set of pre-defined rules to reason and solve problems. The first expert system, MYCIN, was developed in 1976 at Stanford University.\n",
|
||||
"\n",
|
||||
"**Machine Learning (1980s)**\n",
|
||||
"\n",
|
||||
"Machine learning, a subset of AI, emerged in the 1980s. Machine learning algorithms enabled machines to learn from data and improve their performance over time. David Rumelhart, Geoffrey Hinton, and Yann LeCun developed the backpropagation algorithm, a fundamental component of neural networks, in the 1980s.\n",
|
||||
"\n",
|
||||
"**AI Winter (1980s-1990s)**\n",
|
||||
"\n",
|
||||
"Despite the progress made in AI research, the field experienced a decline in funding and interest in the 1980s and 1990s. This period is known as the \"AI Winter.\" The lack of significant breakthroughs and the failure of many AI projects led to a decrease in investment and a shift in focus towards other areas of computer science.\n",
|
||||
"\n",
|
||||
"**Resurgence (2000s)**\n",
|
||||
"\n",
|
||||
"The 21st century saw a resurgence of interest in AI, driven by advances in computing power, data storage, and machine learning algorithms. The development of deep learning techniques, such as convolutional neural networks (CNNs) and recurrent neural networks (RNNs), enabled machines to learn complex patterns in data.\n",
|
||||
"\n",
|
||||
"**Big Data and Deep Learning (2010s)**\n",
|
||||
"\n",
|
||||
"The availability of large datasets and the development of deep learning frameworks like TensorFlow and PyTorch led to significant breakthroughs in AI research. AI applications began to emerge in various industries, including computer vision, natural language processing, and robotics.\n",
|
||||
"\n",
|
||||
"**Current State (2020s)**\n",
|
||||
"\n",
|
||||
"Today, AI is ubiquitous, with applications in:\n",
|
||||
"\n",
|
||||
"1. Virtual assistants (e.g., Siri, Alexa)\n",
|
||||
"2. Image recognition (e.g., facial recognition, object detection)\n",
|
||||
"3. Natural language processing (e.g., language translation, text summarization)\n",
|
||||
"4. Robotics (e.g., autonomous vehicles, robotic process automation)\n",
|
||||
"5. Healthcare (e.g., medical diagnosis, personalized medicine)\n",
|
||||
"\n",
|
||||
"The history of AI is marked by periods of rapid progress, followed by periods of decline and rediscovery. As AI continues to evolve, we can expect to see new breakthroughs and innovations that transform industries and improve our lives.\n",
|
||||
"\n",
|
||||
"**Key Players and Milestones:**\n",
|
||||
"\n",
|
||||
"1. Alan Turing (1950): Proposed the Turing Test\n",
|
||||
"2. John McCarthy (1956): Coined the term \"Artificial Intelligence\"\n",
|
||||
"3. David Rumelhart, Geoffrey Hinton, and Yann LeCun (1980s): Developed the backpropagation algorithm\n",
|
||||
"4. Yann LeCun, Yoshua Bengio, and Geoffrey Hinton (2010s): Developed deep learning techniques\n",
|
||||
"5. Andrew Ng and Fei-Fei Li (2010s): Developed AI applications in computer vision and natural language processing\n",
|
||||
"\n",
|
||||
"**Timeline:**\n",
|
||||
"\n",
|
||||
"* 1950: Alan Turing proposes the Turing Test\n",
|
||||
"* 1956: John McCarthy coins the term \"Artificial Intelligence\"\n",
|
||||
"* 1960s: Rule-based expert systems emerge\n",
|
||||
"* 1980s: Machine learning and AI Winter\n",
|
||||
"* 2000s: Resurgence of interest in AI\n",
|
||||
"* 2010s: Big data and deep learning lead to significant breakthroughs\n",
|
||||
"* 2020s: AI becomes ubiquitous in various industries"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"query = \"write about the history of AI\"\n",
|
||||
"\n",
|
||||
"streaming_response = llm.stream_complete(query)\n",
|
||||
"\n",
|
||||
"for chunk in streaming_response:\n",
|
||||
" try:\n",
|
||||
" print(chunk.raw[\"choices\"][0][\"delta\"][\"content\"], end=\"\", flush=True)\n",
|
||||
" except: \n",
|
||||
" pass"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "base",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Reference in New Issue
Block a user