patchy631--ai-engineering-hub/context-engineering-workflow/src/tools/rag_tool.py

import os
import json
from typing import Dict, Any, List, Optional, Type
from pydantic import BaseModel, Field
from crewai.tools import BaseTool


class RAGInput(BaseModel):
    """Input schema for RAG search tool"""
    query: str = Field(..., description="The search query for retrieval.")
    top_k: int = Field(default=3, description="Maximum number of retrieved results to fetch.")
    document_paths: List[str] = Field(default=None, description="Optional list of document paths to load. Only needed if no documents are already loaded in the vector database.")


class RAGTool(BaseTool):
    name: str = "rag_search"
    description: str = "Search through research documents for relevant information."
    rag_pipeline: Any = Field(..., description="RAG pipeline instance")
    args_schema: Type[BaseModel] = RAGInput
    
    def _run(self, query: str, top_k: int = 3, document_paths: List[str] = None):
        try:
            doc_count = self.rag_pipeline.vector_db.get_collection_count()
            if doc_count == 0:
                if not document_paths:
                    return json.dumps({
                        "status": "INSUFFICIENT_CONTEXT",
                        "source_used": "RAG",
                        "answer": "No documents have been loaded into the RAG system. Please provide document_paths to load documents first, or ensure documents have been previously loaded.",
                        "citations": [],
                        "confidence": 0.0,
                        "retrieval_metadata": {
                            "retrieved_chunks": 0,
                            "top_scores": [],
                            "document_count": 0
                        }
                    })
                
                # load documents
                load_result = self._load_documents(document_paths)
                if load_result["status"] == "ERROR":
                    return json.dumps(load_result, indent=2)
                
                doc_count = self.rag_pipeline.vector_db.get_collection_count()
                if doc_count == 0:
                    return json.dumps({
                        "status": "INSUFFICIENT_CONTEXT",
                        "source_used": "RAG",
                        "answer": "Failed to load documents into the RAG system.",
                        "citations": [],
                        "confidence": 0.0,
                        "retrieval_metadata": {
                            "retrieved_chunks": 0,
                            "top_scores": [],
                            "document_count": 0
                        }
                    })
            
            # Retrieve relevant context (no generation)
            context_results = self.rag_pipeline.retrieve_context(query, top_k=top_k)
            if not context_results:
                return json.dumps({
                    "status": "INSUFFICIENT_CONTEXT",
                    "source_used": "RAG",
                    "answer": f"No relevant context found for query: '{query}'",
                    "citations": [],
                    "confidence": 0.0,
                    "retrieval_metadata": {
                        "retrieved_chunks": 0,
                        "top_scores": [],
                        "document_count": doc_count
                    }
                })
            
            context_blocks = []
            citations = []
            for i, result in enumerate(context_results):
                chunk_text = result.get("text", "")
                score = result.get("score", 0.0)
                page_number = result.get("page_number", 0)
                chunk_index = result.get("chunk_index", i)
                source_file = result.get("source_file", "unknown")
                
                filename = source_file.split("/")[-1] if source_file != "unknown" else "unknown"
                context_blocks.append(f"**Context {i+1} (Score: {score:.3f}, Page {page_number}, Chunk {chunk_index})**\n{chunk_text[:500]}...")
                citations.append({
                    "label": f"{filename} - Page {page_number}, Chunk {chunk_index}",
                    "locator": f"page_{page_number}_chunk_{chunk_index}",
                    "page_number": page_number,
                    "chunk_index": chunk_index,
                    "source_file": filename,
                    "score": score,
                    "content": chunk_text
                })
            
            answer = "\n\n".join(context_blocks)
            
            return json.dumps({
                "status": "OK",
                "source_used": "RAG",
                "answer": f"Retrieved {len(context_results)} relevant context chunks:\n\n{answer}",
                "citations": citations,
                "confidence": max([r.get("score", 0.0) for r in context_results]),
                "retrieval_metadata": {
                    "retrieved_chunks": len(context_results),
                    "top_scores": [r.get("score", 0.0) for r in context_results],
                    "document_count": doc_count
                },
                "raw_context": context_results  # Include raw context for further processing
            })
            
        except Exception as e:
            return json.dumps({
                "status": "ERROR",
                "source_used": "RAG",
                "answer": f"RAG search failed: {str(e)}",
                "citations": [],
                "confidence": 0.0,
                "retrieval_metadata": {
                    "retrieved_chunks": 0,
                    "top_scores": [],
                    "document_count": 0
                },
                "error": str(e)
            })
    
    def _load_documents(self, document_paths: List[str]) -> Dict[str, Any]:
        """Helper method to load documents into the RAG pipeline"""
        try:
            if not document_paths:
                return {
                    "status": "ERROR",
                    "source_used": "RAG",
                    "answer": "No document paths provided",
                    "citations": [],
                    "confidence": 0.0
                }
            
            missing_files = [path for path in document_paths if not os.path.exists(path)]
            if missing_files:
                return {
                    "status": "ERROR",
                    "source_used": "RAG",
                    "answer": f"Missing files: {missing_files}",
                    "citations": [],
                    "confidence": 0.0
                }
            
            results = self.rag_pipeline.process_documents(document_paths)
            return {
                "status": "OK",
                "source_used": "RAG",
                "answer": f"Successfully processed {len(results['processed_docs'])} documents with {results['total_chunks']} total chunks.",
                "citations": [{"label": f"Processed: {doc['path']}", "locator": doc['path']} for doc in results['processed_docs']],
                "confidence": 0.96
            }
            
        except Exception as e:
            return {
                "status": "ERROR",
                "source_used": "RAG",
                "answer": f"Document processing failed: {str(e)}",
                "citations": [],
                "confidence": 0.0,
                "error": str(e)
            }