chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1 @@
|
||||
OPENAI_API_KEY="your-openai-api-key"
|
||||
@@ -0,0 +1,43 @@
|
||||
|
||||
# 100% local RAG app to chat with GitHub!
|
||||
|
||||
This project leverages GitIngest to parse a GitHub repo in markdown format and the use LlamaIndex for RAG orchestration over it.
|
||||
|
||||
|
||||
## Installation and setup
|
||||
|
||||
**Install Dependencies**:
|
||||
Ensure you have Python 3.9 or later installed (tested with Python 3.11.9).
|
||||
|
||||
**Option 1: Using requirements.txt (Recommended)**
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
**Option 2: Manual installation**
|
||||
```bash
|
||||
pip install gitingest llama-index llama-index-llms-ollama llama-index-llms-openai llama-index-agent-openai llama-index-embeddings-huggingface streamlit pandas python-dotenv huggingface-hub
|
||||
```
|
||||
|
||||
**Environment Setup**:
|
||||
For OpenAI integration, create a `.env` file in the project directory:
|
||||
```
|
||||
OPENAI_API_KEY=your_openai_api_key_here
|
||||
```
|
||||
|
||||
**Running**:
|
||||
|
||||
Make sure you have Ollama Server running then you can run following command to start the streamlit application ```streamlit run app_local.py```.
|
||||
|
||||
---
|
||||
|
||||
## 📬 Stay Updated with Our Newsletter!
|
||||
**Get a FREE Data Science eBook** 📖 with 150+ essential lessons in Data Science when you subscribe to our newsletter! Stay in the loop with the latest tutorials, insights, and exclusive resources. [Subscribe now!](https://join.dailydoseofds.com)
|
||||
|
||||
[](https://join.dailydoseofds.com)
|
||||
|
||||
---
|
||||
|
||||
## Contribution
|
||||
|
||||
Contributions are welcome! Please fork the repository and submit a pull request with your improvements.
|
||||
@@ -0,0 +1,226 @@
|
||||
import os
|
||||
import gc
|
||||
import tempfile
|
||||
import uuid
|
||||
import pandas as pd
|
||||
from typing import Optional, Dict, Any
|
||||
import logging
|
||||
|
||||
from gitingest import ingest
|
||||
from llama_index.core import Settings, PromptTemplate, VectorStoreIndex, SimpleDirectoryReader
|
||||
from llama_index.core.node_parser import MarkdownNodeParser
|
||||
import streamlit as st
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Constants
|
||||
MAX_REPO_SIZE = 100 * 1024 * 1024 # 100MB
|
||||
SUPPORTED_REPO_TYPES = ['.py', '.md', '.ipynb', '.js', '.ts', '.json']
|
||||
|
||||
class GitHubRAGError(Exception):
|
||||
"""Custom exception for GitHub RAG application errors"""
|
||||
pass
|
||||
|
||||
def validate_github_url(url: str) -> bool:
|
||||
"""Validate GitHub repository URL"""
|
||||
return url.startswith(('https://github.com/', 'http://github.com/'))
|
||||
|
||||
def get_repo_name(url: str) -> str:
|
||||
"""Extract repository name from URL"""
|
||||
try:
|
||||
return url.split('/')[-1].replace('.git', '')
|
||||
except Exception as e:
|
||||
raise GitHubRAGError(f"Invalid repository URL: {str(e)}")
|
||||
|
||||
def reset_chat():
|
||||
"""Reset chat session and clean up resources"""
|
||||
try:
|
||||
st.session_state.messages = []
|
||||
st.session_state.context = None
|
||||
gc.collect()
|
||||
logger.info("Chat session reset successfully")
|
||||
except Exception as e:
|
||||
logger.error(f"Error resetting chat: {str(e)}")
|
||||
raise GitHubRAGError("Failed to reset chat session")
|
||||
|
||||
def process_with_gitingets(github_url: str) -> tuple:
|
||||
"""Process GitHub repository using gitingest"""
|
||||
try:
|
||||
summary, tree, content = ingest(github_url)
|
||||
if not all([summary, tree, content]):
|
||||
raise GitHubRAGError("Failed to process repository: Missing data")
|
||||
return summary, tree, content
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing repository: {str(e)}")
|
||||
raise GitHubRAGError(f"Failed to process repository: {str(e)}")
|
||||
|
||||
def create_query_engine(content_path: str, repo_name: str) -> Any:
|
||||
"""Create and configure query engine"""
|
||||
try:
|
||||
loader = SimpleDirectoryReader(input_dir=content_path)
|
||||
docs = loader.load_data()
|
||||
node_parser = MarkdownNodeParser()
|
||||
index = VectorStoreIndex.from_documents(
|
||||
documents=docs,
|
||||
transformations=[node_parser],
|
||||
show_progress=True
|
||||
)
|
||||
|
||||
qa_prompt_tmpl_str = """
|
||||
You are an AI assistant specialized in analyzing GitHub repositories.
|
||||
|
||||
Repository structure:
|
||||
{tree}
|
||||
---------------------
|
||||
|
||||
Context information from the repository:
|
||||
{context_str}
|
||||
---------------------
|
||||
|
||||
Given the repository structure and context above, provide a clear and precise answer to the query.
|
||||
Focus on the repository's content, code structure, and implementation details.
|
||||
If the information is not available in the context, respond with 'I don't have enough information about that aspect of the repository.'
|
||||
|
||||
Query: {query_str}
|
||||
Answer: """
|
||||
|
||||
qa_prompt_tmpl = PromptTemplate(qa_prompt_tmpl_str)
|
||||
query_engine = index.as_query_engine(streaming=True)
|
||||
query_engine.update_prompts(
|
||||
{"response_synthesizer:text_qa_template": qa_prompt_tmpl}
|
||||
)
|
||||
return query_engine
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating query engine: {str(e)}")
|
||||
raise GitHubRAGError(f"Failed to create query engine: {str(e)}")
|
||||
|
||||
# Initialize session state
|
||||
if "id" not in st.session_state:
|
||||
st.session_state.id = uuid.uuid4()
|
||||
st.session_state.file_cache = {}
|
||||
st.session_state.messages = []
|
||||
|
||||
session_id = st.session_state.id
|
||||
|
||||
# Sidebar
|
||||
with st.sidebar:
|
||||
st.header("Add your GitHub repository!")
|
||||
|
||||
github_url = st.text_input(
|
||||
"Enter GitHub repository URL",
|
||||
placeholder="https://github.com/username/repo",
|
||||
help="Enter a valid GitHub repository URL"
|
||||
)
|
||||
|
||||
load_repo = st.button("Load Repository", type="primary")
|
||||
|
||||
if github_url and load_repo:
|
||||
try:
|
||||
# Validate URL
|
||||
if not validate_github_url(github_url):
|
||||
st.error("Please enter a valid GitHub repository URL")
|
||||
st.stop()
|
||||
|
||||
repo_name = get_repo_name(github_url)
|
||||
file_key = f"{session_id}-{repo_name}"
|
||||
|
||||
if file_key not in st.session_state.file_cache:
|
||||
with st.spinner("Processing your repository..."):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
try:
|
||||
summary, tree, content = process_with_gitingets(github_url)
|
||||
|
||||
# Write content to temporary file
|
||||
content_path = os.path.join(temp_dir, f"{repo_name}_content.md")
|
||||
with open(content_path, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
# Create and cache query engine
|
||||
query_engine = create_query_engine(temp_dir, repo_name)
|
||||
st.session_state.file_cache[file_key] = query_engine
|
||||
|
||||
st.success("Repository loaded successfully! Ready to chat.")
|
||||
logger.info(f"Successfully processed repository: {repo_name}")
|
||||
|
||||
except GitHubRAGError as e:
|
||||
st.error(str(e))
|
||||
logger.error(f"Error processing repository {repo_name}: {str(e)}")
|
||||
st.stop()
|
||||
except Exception as e:
|
||||
st.error("An unexpected error occurred while processing the repository")
|
||||
logger.error(f"Unexpected error: {str(e)}")
|
||||
st.stop()
|
||||
else:
|
||||
st.info("Repository already loaded. Ready to chat!")
|
||||
|
||||
except Exception as e:
|
||||
st.error(f"An error occurred: {str(e)}")
|
||||
logger.error(f"Error in repository loading process: {str(e)}")
|
||||
st.stop()
|
||||
|
||||
# Main content
|
||||
col1, col2 = st.columns([6, 1])
|
||||
|
||||
with col1:
|
||||
st.header("Chat with GitHub using RAG </>")
|
||||
|
||||
with col2:
|
||||
st.button("Clear Chat ↺", on_click=reset_chat, help="Clear chat history and reset session")
|
||||
|
||||
# Display chat history
|
||||
for message in st.session_state.messages:
|
||||
with st.chat_message(message["role"]):
|
||||
st.markdown(message["content"])
|
||||
|
||||
# Chat input
|
||||
if prompt := st.chat_input("What's up?"):
|
||||
try:
|
||||
# Add user message to chat history
|
||||
st.session_state.messages.append({"role": "user", "content": prompt})
|
||||
|
||||
# Display user message
|
||||
with st.chat_message("user"):
|
||||
st.markdown(prompt)
|
||||
|
||||
# Process and display assistant response
|
||||
with st.chat_message("assistant"):
|
||||
message_placeholder = st.empty()
|
||||
full_response = ""
|
||||
|
||||
try:
|
||||
repo_name = get_repo_name(github_url)
|
||||
file_key = f"{session_id}-{repo_name}"
|
||||
query_engine = st.session_state.file_cache.get(file_key)
|
||||
|
||||
if query_engine is None:
|
||||
raise GitHubRAGError("Please load a repository first!")
|
||||
|
||||
response = query_engine.query(prompt)
|
||||
|
||||
if hasattr(response, 'response_gen'):
|
||||
for chunk in response.response_gen:
|
||||
if isinstance(chunk, str):
|
||||
full_response += chunk
|
||||
message_placeholder.markdown(full_response + "▌")
|
||||
else:
|
||||
full_response = str(response)
|
||||
message_placeholder.markdown(full_response)
|
||||
|
||||
message_placeholder.markdown(full_response)
|
||||
st.session_state.messages.append({"role": "assistant", "content": full_response})
|
||||
|
||||
except GitHubRAGError as e:
|
||||
st.error(str(e))
|
||||
logger.error(f"Error in chat processing: {str(e)}")
|
||||
except Exception as e:
|
||||
st.error("An unexpected error occurred while processing your query")
|
||||
logger.error(f"Unexpected error in chat: {str(e)}")
|
||||
|
||||
except Exception as e:
|
||||
st.error("An error occurred in the chat system")
|
||||
logger.error(f"Chat system error: {str(e)}")
|
||||
@@ -0,0 +1,180 @@
|
||||
import os
|
||||
|
||||
import gc
|
||||
import tempfile
|
||||
import uuid
|
||||
import pandas as pd
|
||||
|
||||
from gitingest import ingest
|
||||
|
||||
from llama_index.core import Settings
|
||||
from llama_index.llms.ollama import Ollama
|
||||
from llama_index.core import PromptTemplate
|
||||
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
|
||||
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
|
||||
from llama_index.core.node_parser import MarkdownNodeParser
|
||||
|
||||
import streamlit as st
|
||||
|
||||
if "id" not in st.session_state:
|
||||
st.session_state.id = uuid.uuid4()
|
||||
st.session_state.file_cache = {}
|
||||
|
||||
session_id = st.session_state.id
|
||||
client = None
|
||||
|
||||
@st.cache_resource
|
||||
def load_llm():
|
||||
llm = Ollama(model="llama3.2", request_timeout=120.0)
|
||||
return llm
|
||||
|
||||
def reset_chat():
|
||||
st.session_state.messages = []
|
||||
st.session_state.context = None
|
||||
gc.collect()
|
||||
|
||||
def process_with_gitingets(github_url):
|
||||
# or from URL
|
||||
summary, tree, content = ingest(github_url)
|
||||
return summary, tree, content
|
||||
|
||||
|
||||
with st.sidebar:
|
||||
st.header(f"Add your GitHub repository!")
|
||||
|
||||
github_url = st.text_input("Enter GitHub repository URL", placeholder="GitHub URL")
|
||||
load_repo = st.button("Load Repository")
|
||||
|
||||
if github_url and load_repo:
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
st.write("Processing your repository...")
|
||||
repo_name = github_url.split('/')[-1]
|
||||
file_key = f"{session_id}-{repo_name}"
|
||||
|
||||
if file_key not in st.session_state.get('file_cache', {}):
|
||||
|
||||
if os.path.exists(temp_dir):
|
||||
summary, tree, content = process_with_gitingets(github_url)
|
||||
|
||||
# Write summary to a markdown file
|
||||
with open("content.md", "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
# Write summary to a markdown file in temp directory
|
||||
content_path = os.path.join(temp_dir, f"{repo_name}_content.md")
|
||||
with open(content_path, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
loader = SimpleDirectoryReader(
|
||||
input_dir=temp_dir,
|
||||
)
|
||||
else:
|
||||
st.error('Could not find the file you uploaded, please check again...')
|
||||
st.stop()
|
||||
|
||||
docs = loader.load_data()
|
||||
|
||||
# setup llm & embedding model
|
||||
llm=load_llm()
|
||||
embed_model = HuggingFaceEmbedding( model_name="BAAI/bge-large-en-v1.5", trust_remote_code=True)
|
||||
# Creating an index over loaded data
|
||||
Settings.embed_model = embed_model
|
||||
node_parser = MarkdownNodeParser()
|
||||
index = VectorStoreIndex.from_documents(documents=docs, transformations=[node_parser], show_progress=True)
|
||||
|
||||
# Create the query engine, where we use a cohere reranker on the fetched nodes
|
||||
Settings.llm = llm
|
||||
query_engine = index.as_query_engine(streaming=True)
|
||||
|
||||
# ====== Customise prompt template ======
|
||||
qa_prompt_tmpl_str = (
|
||||
"Context information is below.\n"
|
||||
"---------------------\n"
|
||||
"{context_str}\n"
|
||||
"---------------------\n"
|
||||
"Given the context information above I want you to think step by step to answer the query in a highly precise and crisp manner focused on the final answer, incase case you don't know the answer say 'I don't know!'.\n"
|
||||
"Query: {query_str}\n"
|
||||
"Answer: "
|
||||
)
|
||||
qa_prompt_tmpl = PromptTemplate(qa_prompt_tmpl_str)
|
||||
|
||||
query_engine.update_prompts(
|
||||
{"response_synthesizer:text_qa_template": qa_prompt_tmpl}
|
||||
)
|
||||
|
||||
st.session_state.file_cache[file_key] = query_engine
|
||||
else:
|
||||
query_engine = st.session_state.file_cache[file_key]
|
||||
|
||||
# Inform the user that the file is processed and Display the PDF uploaded
|
||||
st.success("Ready to Chat!")
|
||||
except Exception as e:
|
||||
st.error(f"An error occurred: {e}")
|
||||
st.stop()
|
||||
|
||||
col1, col2 = st.columns([6, 1])
|
||||
|
||||
with col1:
|
||||
st.header(f"Chat with GitHub using RAG </>")
|
||||
|
||||
with col2:
|
||||
st.button("Clear ↺", on_click=reset_chat)
|
||||
|
||||
# Initialize chat history
|
||||
if "messages" not in st.session_state:
|
||||
reset_chat()
|
||||
|
||||
|
||||
# Display chat messages from history on app rerun
|
||||
for message in st.session_state.messages:
|
||||
with st.chat_message(message["role"]):
|
||||
st.markdown(message["content"])
|
||||
|
||||
|
||||
# Accept user input
|
||||
if prompt := st.chat_input("What's up?"):
|
||||
# Add user message to chat history
|
||||
st.session_state.messages.append({"role": "user", "content": prompt})
|
||||
# Display user message in chat message container
|
||||
with st.chat_message("user"):
|
||||
st.markdown(prompt)
|
||||
|
||||
# Display assistant response in chat message container
|
||||
with st.chat_message("assistant"):
|
||||
message_placeholder = st.empty()
|
||||
full_response = ""
|
||||
|
||||
try:
|
||||
# Get the repo name from the GitHub URL
|
||||
repo_name = github_url.split('/')[-1]
|
||||
file_key = f"{session_id}-{repo_name}"
|
||||
|
||||
# Get query engine from session state
|
||||
query_engine = st.session_state.file_cache.get(file_key)
|
||||
|
||||
if query_engine is None:
|
||||
st.error("Please load a repository first!")
|
||||
st.stop()
|
||||
|
||||
# Use the query engine
|
||||
response = query_engine.query(prompt)
|
||||
|
||||
# Handle streaming response
|
||||
if hasattr(response, 'response_gen'):
|
||||
for chunk in response.response_gen:
|
||||
if isinstance(chunk, str): # Only process string chunks
|
||||
full_response += chunk
|
||||
message_placeholder.markdown(full_response + "▌")
|
||||
else:
|
||||
# Handle non-streaming response
|
||||
full_response = str(response)
|
||||
message_placeholder.markdown(full_response)
|
||||
|
||||
message_placeholder.markdown(full_response)
|
||||
except Exception as e:
|
||||
st.error(f"An error occurred while processing your query: {str(e)}")
|
||||
full_response = "Sorry, I encountered an error while processing your request."
|
||||
message_placeholder.markdown(full_response)
|
||||
|
||||
# Add assistant response to chat history
|
||||
st.session_state.messages.append({"role": "assistant", "content": full_response})
|
||||
@@ -0,0 +1,18 @@
|
||||
# GitHub RAG Application Requirements
|
||||
# Supports Python >=3.9, <4.0
|
||||
|
||||
# Core dependencies
|
||||
gitingest
|
||||
streamlit
|
||||
python-dotenv
|
||||
pandas
|
||||
|
||||
# LlamaIndex core and integrations - using compatible versions
|
||||
llama-index
|
||||
llama-index-llms-ollama
|
||||
llama-index-llms-openai
|
||||
llama-index-agent-openai
|
||||
llama-index-embeddings-huggingface
|
||||
|
||||
# Additional dependencies for local model support
|
||||
huggingface-hub
|
||||
Reference in New Issue
Block a user