chore: import upstream snapshot with attribution

examples/eval-rag-full/.gitignore

@@ -0,0 +1,3 @@
+venv
+__pycache__
+db

examples/eval-rag-full/README.md

@@ -0,0 +1,47 @@
+# eval-rag-full (Rag Full)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example eval-rag-full
+cd eval-rag-full
+```
+
+## Usage
+
+This RAG example allows you to ask questions over a number of public company SEC filings. It uses LangChain, but the flow is representative of any RAG solution.
+
+There are 3 parts:
+
+1. `ingest.py`: Chunks and loads PDFs into a vector database (PDFs are pulled from a public Google Cloud bucket)
+
+1. `retrieve.py`: Promptfoo-compatible provider that answers RAG questions using the database.
+
+1. `promptfooconfig.yaml`: Test inputs and requirements.
+
+To get started:
+
+1. Set the OPENAI_API_KEY environment variable.
+
+1. Create a python virtual environment: `python3 -m venv venv`
+
+1. Enter the environment: `source venv/bin/activate`
+
+1. Install python dependencies: `pip install -r requirements.txt`
+
+1. Run `ingest.py` to create the vector database: `python ingest.py`
+
+Now we're ready to go.
+
+- Edit `promptfooconfig.yaml` to your liking to configure the questions you'd like to ask in your tests. Then run:
+- Edit `retrieve.py` to control how context is loaded and questions are answered.
+
+```bash
+npx promptfoo@latest eval
+```
+
+Promptfoo is a Node.js CLI, but the `file://retrieve.py` provider runs inside Python. Keep the virtual environment active when running the eval, or set `PROMPTFOO_PYTHON=./venv/bin/python` so Promptfoo can import the packages from `requirements.txt`.
+
+Afterwards, you can view the results by running `npx promptfoo@latest view`
+
+See `promptfooconfig.with-asserts.yaml` for a more complete example that compares the performance of two RAG configurations. The smaller retrieval configuration is intentionally expected to miss a couple of details so the comparison view demonstrates failures as well as passes.

examples/eval-rag-full/ingest.py

@@ -0,0 +1,167 @@
+"""
+PDF document ingestion script for RAG implementation.
+Loads PDF files from a remote source in parallel, splits them into chunks,
+and stores them in a Chroma vector database.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import logging
+import os
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+from urllib.parse import quote
+
+from langchain_chroma import Chroma
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_core.documents import Document
+from langchain_openai import OpenAIEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from tqdm import tqdm
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+
+# Constants
+OPENAI_API_KEY: Optional[str] = os.getenv("OPENAI_API_KEY")
+if not OPENAI_API_KEY:
+    raise ValueError("OPENAI_API_KEY environment variable is not set")
+
+EXAMPLE_DIR: Path = Path(__file__).resolve().parent
+CHROMA_PATH: str = str(EXAMPLE_DIR / "db")
+BASE_URL: str = "https://storage.googleapis.com/promptfoo-public-1/examples/rag-sec/"
+CHUNK_SIZE: int = 500
+CHUNK_OVERLAP: int = 50
+MAX_WORKERS: int = 5
+OPENAI_AI_EMBEDDING_MODEL: str = "text-embedding-3-large"
+
+# List of PDF files to process
+PDF_FILES: List[str] = [
+    "2022 Q3 AAPL.pdf",
+    "2022 Q3 AMZN.pdf",
+    "2022 Q3 INTC.pdf",
+    "2022 Q3 MSFT.pdf",
+    "2022 Q3 NVDA.pdf",
+    "2023 Q1 AAPL.pdf",
+    "2023 Q1 AMZN.pdf",
+    "2023 Q1 INTC.pdf",
+    "2023 Q1 MSFT.pdf",
+    "2023 Q1 NVDA.pdf",
+    "2023 Q2 AAPL.pdf",
+    "2023 Q2 AMZN.pdf",
+    "2023 Q2 INTC.pdf",
+    "2023 Q2 MSFT.pdf",
+    "2023 Q2 NVDA.pdf",
+    "2023 Q3 AAPL.pdf",
+    "2023 Q3 AMZN.pdf",
+    "2023 Q3 INTC.pdf",
+    "2023 Q3 MSFT.pdf",
+    "2023 Q3 NVDA.pdf",
+]
+
+
+def process_single_pdf(pdf_file: str) -> Tuple[str, List[Document]]:
+    """
+    Process a single PDF file and return its chunks.
+
+    Args:
+        pdf_file: Name of the PDF file to process
+
+    Returns:
+        Tuple containing filename and list of document chunks
+    """
+    doc_url: str = BASE_URL + quote(pdf_file)
+    try:
+        loader: PyPDFLoader = PyPDFLoader(doc_url)
+        pages: List[Document] = loader.load()
+        text_splitter: RecursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(
+            chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP
+        )
+        chunks: List[Document] = text_splitter.split_documents(pages)
+        return pdf_file, chunks
+    except Exception as e:
+        logging.error(f"Error processing {pdf_file}: {str(e)}")
+        return pdf_file, []
+
+
+def process_pdfs() -> List[Document]:
+    """
+    Process PDF files from the remote source in parallel and split them into chunks.
+
+    Returns:
+        List of document chunks
+    """
+    all_chunks: List[Document] = []
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+        # Submit all PDF processing tasks
+        future_to_pdf: Dict[
+            concurrent.futures.Future[Tuple[str, List[Document]]], str
+        ] = {
+            executor.submit(process_single_pdf, pdf_file): pdf_file
+            for pdf_file in PDF_FILES
+        }
+
+        # Process completed tasks with progress bar
+        with tqdm(total=len(PDF_FILES), desc="Processing PDFs") as pbar:
+            for future in concurrent.futures.as_completed(future_to_pdf):
+                pdf_file: str = future_to_pdf[future]
+                try:
+                    _, chunks = future.result()
+                    all_chunks.extend(chunks)
+                    pbar.update(1)
+                except Exception as e:
+                    logging.error(f"Failed to process {pdf_file}: {str(e)}")
+                    pbar.update(1)
+
+    logging.info(f"Processed {len(all_chunks)} chunks from {len(PDF_FILES)} files")
+    return all_chunks
+
+
+def create_vector_store(chunks: List[Document], batch_size: int = 100) -> None:
+    """
+    Create and persist the vector store from document chunks in batches.
+
+    Args:
+        chunks: List of document chunks to embed
+        batch_size: Number of documents to process in each batch
+    """
+    embeddings: OpenAIEmbeddings = OpenAIEmbeddings(
+        model=OPENAI_AI_EMBEDDING_MODEL, openai_api_key=OPENAI_API_KEY
+    )
+
+    logging.info("Creating vector store...")
+
+    # Process first batch
+    current_batch: List[Document] = chunks[:batch_size]
+    db: Chroma = Chroma.from_documents(
+        current_batch,
+        embeddings,
+        persist_directory=CHROMA_PATH,
+        collection_name="rag_collection",
+    )
+
+    # Process remaining batches
+    with tqdm(
+        total=len(chunks), initial=batch_size, desc="Embedding documents"
+    ) as pbar:
+        for i in range(batch_size, len(chunks), batch_size):
+            current_batch = chunks[i : i + batch_size]
+            db.add_documents(current_batch)
+            pbar.update(len(current_batch))
+
+    logging.info(f"Vector store created and persisted to {CHROMA_PATH}")
+
+
+def main() -> None:
+    """Main execution function."""
+    chunks: List[Document] = process_pdfs()
+    if chunks:
+        create_vector_store(chunks)
+
+
+if __name__ == "__main__":
+    main()

examples/eval-rag-full/promptfooconfig.with-asserts.yaml

@@ -0,0 +1,65 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'RAG - End to end test with comparison'
+
+prompts:
+  - '{{question}}'
+
+providers:
+  - id: file://retrieve.py
+    label: RAG retrieval small
+    config:
+      topK: 5
+  - id: file://retrieve.py
+    label: RAG retrieval big
+    config:
+      topK: 10
+
+tests:
+  - vars:
+      question: How has Apple's total net sales changed over time?
+    assert:
+      - type: contains
+        value: '82,959 million'
+      - type: contains
+        value: '304,182 million'
+  - vars:
+      question: What are the major factors contributing to the change in Apple's gross margin in the most recent 10-Q compared to the previous quarters?
+    assert:
+      - type: llm-rubric
+        value: 'cites weakness in foreign currencies relative to the U.S. dollar'
+  - vars:
+      question: Has there been any significant change in Microsoft's operating expenses over the reported quarters? If so, what are the key drivers for this change?
+    assert:
+      - type: llm-rubric
+        value: 'mentions investments in cloud engineering and employee severance expenses'
+  - vars:
+      question: How has Apple's revenue from iPhone sales fluctuated across quarters?
+    assert:
+      - type: llm-rubric
+        value: 'describes iPhone net sales in 2023 compared with 2022'
+      - type: contains
+        value: 'iPhone 14 Pro'
+  - vars:
+      question: Can any trends be identified in Apple's Services segment revenue over the reported periods?
+    assert:
+      - type: icontains
+        value: services
+  - vars:
+      question: What is the impact of foreign exchange rates on Apple's financial performance? List this out separately for each reported period.
+    assert:
+      - type: llm-rubric
+        value: 'includes specific commentary on foreign exchange'
+      - type: contains
+        value: 'Q3 2023'
+  - vars:
+      question: Are there any notable changes in Apple's liquidity position or cash flows as reported in these 10-Qs?
+    assert:
+      - type: llm-rubric
+        value: 'mentions decreases in cash and net income'
+  - vars:
+      question: Examine how Intel's effective tax rate in the most recent 10-Q compares with the tax-related discussions in the notes section.
+    assert:
+      - type: contains
+        value: '18%'
+  - vars:
+      question: In Amazon's latest 10-Q, how does the revenue distribution across its diverse business segments like e-commerce, AWS, and others compare to the costs incurred in these segments?

examples/eval-rag-full/promptfooconfig.yaml

@@ -0,0 +1,28 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'RAG - End to end test'
+
+prompts:
+  - '{{question}}'
+
+providers:
+  - file://retrieve.py
+
+tests:
+  - vars:
+      question: How has Apple's total net sales changed over time?
+  - vars:
+      question: What are the major factors contributing to the change in Apple's gross margin in the most recent 10-Q compared to the previous quarters?
+  - vars:
+      question: Has there been any significant change in Microsoft's operating expenses over the reported quarters? If so, what are the key drivers for this change?
+  - vars:
+      question: How has Apple's revenue from iPhone sales fluctuated across quarters?
+  - vars:
+      question: Can any trends be identified in Apple's Services segment revenue over the reported periods?
+  - vars:
+      question: What is the impact of foreign exchange rates on Apple's financial performance? List this out separately for each reported period.
+  - vars:
+      question: Are there any notable changes in Apple's liquidity position or cash flows as reported in these 10-Qs?
+  - vars:
+      question: Examine how Intel's effective tax rate in the most recent 10-Q compares with the tax-related discussions in the notes section.
+  - vars:
+      question: In Amazon's latest 10-Q, how does the revenue distribution across its diverse business segments like e-commerce, AWS, and others compare to the costs incurred in these segments?

examples/eval-rag-full/requirements.txt

@@ -0,0 +1,8 @@
+chromadb==1.5.9
+langchain-chroma==1.1.0
+langchain-community==0.4.1
+langchain-core==1.3.3
+langchain-openai==1.2.1
+langchain-text-splitters==1.1.2
+pypdf==6.13.3
+tqdm==4.67.3

examples/eval-rag-full/retrieve.py

@@ -0,0 +1,92 @@
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Tuple
+
+from langchain_chroma import Chroma
+from langchain_core.documents import Document
+from langchain_core.messages import AIMessage, HumanMessage
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+
+# Constants
+EXAMPLE_DIR: Path = Path(__file__).resolve().parent
+CHROMA_PATH: str = str(EXAMPLE_DIR / "db")
+OPENAI_AI_MODEL: str = "gpt-4.1-mini"
+OPENAI_API_KEY: str | None = os.getenv("OPENAI_API_KEY")
+OPENAI_AI_EMBEDDING_MODEL: str = "text-embedding-3-large"
+
+# Initialize embeddings and load the Chroma database
+embeddings: OpenAIEmbeddings = OpenAIEmbeddings(
+    model=OPENAI_AI_EMBEDDING_MODEL, openai_api_key=OPENAI_API_KEY
+)
+db_chroma: Chroma = Chroma(
+    collection_name="rag_collection",
+    persist_directory=CHROMA_PATH,
+    embedding_function=embeddings,
+)
+
+# Prompt template for generating answers
+PROMPT_TEMPLATE: str = """
+Answer the question based only on the following context:
+{context}
+Answer the question based on the above context: {question}.
+Provide a detailed answer.
+Don't justify your answers.
+Don't give information not mentioned in the CONTEXT INFORMATION.
+Do not say "according to the context" or "mentioned in the context" or similar.
+"""
+
+
+def call_api(
+    prompt: str, options: Dict[str, Any], context: Dict[str, Any]
+) -> Dict[str, str]:
+    """
+    Process a prompt using RAG and return the response.
+
+    Args:
+        prompt: The user's question or prompt
+        options: Configuration options including topK
+        context: Additional context for the request
+
+    Returns:
+        Dict containing the model's response
+
+    Raises:
+        Exception: If there's an error during processing
+    """
+    try:
+        k: int = options.get("config", {}).get("topK", 5)
+        docs_chroma: List[Tuple[Document, float]] = (
+            db_chroma.similarity_search_with_score(
+                prompt,
+                k=k,
+            )
+        )
+        context_text: str = "\n\n".join(
+            [doc.page_content for doc, _score in docs_chroma]
+        )
+
+        # Generate prompt using the template
+        prompt_template: ChatPromptTemplate = ChatPromptTemplate.from_template(
+            PROMPT_TEMPLATE
+        )
+        final_prompt: str = prompt_template.format(
+            context=context_text, question=prompt
+        )
+
+        # Fetch from OpenAI API
+        chat: ChatOpenAI = ChatOpenAI(
+            model_name=OPENAI_AI_MODEL, temperature=0, openai_api_key=OPENAI_API_KEY
+        )
+        message: HumanMessage = HumanMessage(content=final_prompt)
+        response: AIMessage = chat.invoke([message])
+
+        result: Dict[str, str] = {
+            "output": response.content,
+        }
+
+        return result
+    except Exception as e:
+        logging.error(f"Error in call_api: {str(e)}")
+        raise