chore: import upstream snapshot with attribution

examples/integration-langgraph/README.md

@@ -0,0 +1,67 @@
+# integration-langgraph (LangGraph Integration)
+
+This example demonstrates how to use LangGraph with Promptfoo, including a research agent setup, structured output, and red teaming or evaluation.
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example integration-langgraph
+cd integration-langgraph
+```
+
+## Environment Variables
+
+This example requires the following environment variables:
+
+- `OPENAI_API_KEY` – Your OpenAI API key (required by LangGraph to use ChatOpenAI)
+
+You can set this in a `.env` file or directly in your environment.
+
+## Prerequisites
+
+- Python 3.9-3.12 tested
+- Node.js v22 LTS or newer
+- OpenAI API access (for GPT-4o, GPT-4o-mini, and OpenAI's forthcoming o3 mini once released)
+- An OpenAI API key
+
+Install Python packages:
+
+```bash
+pip install -r requirements.txt
+```
+
+Or install individually:
+
+```bash
+pip install langgraph langchain langchain-openai python-dotenv
+```
+
+Install promptfoo CLI:
+
+```bash
+npm install -g promptfoo
+```
+
+## Files
+
+- `agent.py`: Defines the LangGraph Research Agent, using a StateGraph that processes user queries and summarizes AI research trends.
+- `provider.py`: Wraps the agent logic into a callable function for Promptfoo, exposing a call_api() handler.
+- `promptfooconfig.yaml`: Configures Promptfoo to:
+
+- Provide test prompts
+- Call the LangGraph provider
+- Check outputs using assertions
+
+Run the evaluation:
+
+```bash
+npx promptfoo eval
+```
+
+Explore results in browser:
+
+```bash
+npx promptfoo view
+```
+
+---

examples/integration-langgraph/agent.py

@@ -0,0 +1,71 @@
+import asyncio
+import os
+
+from langchain_openai import ChatOpenAI
+from langgraph.graph import StateGraph
+from pydantic import BaseModel
+
+# Load the OpenAI API key from environment variable
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+
+
+# Define the data structure (state) passed between nodes in the graph
+class ResearchState(BaseModel):
+    query: str  # The original research query
+    raw_info: str = ""  # Raw fetched or mocked information
+    summary: str = ""  # Final summarized result
+
+
+# Function to create and return the research agent graph
+def get_research_agent(model="gpt-4o"):
+    # Initialize the OpenAI LLM with the specified model and API key
+    llm = ChatOpenAI(model=model, api_key=OPENAI_API_KEY)
+
+    # Create a stateful graph with ResearchState as the shared state type
+    graph = StateGraph(ResearchState)
+
+    # Node 1: Simulate a search function that populates raw_info
+    def search_info(state: ResearchState) -> ResearchState:
+        # TODO: Replace with real search API integration
+        mock_info = f"(Mock) According to recent sources, the latest trends in {state.query} include X, Y, Z."
+        return ResearchState(query=state.query, raw_info=mock_info)
+
+    # Node 2: Use the LLM to summarize the raw_info content
+    def summarize_info(state: ResearchState) -> ResearchState:
+        prompt = f"Summarize the following:\n{state.raw_info}"
+        response = llm.invoke(prompt)  # Call the LLM to get the summary
+        return ResearchState(
+            query=state.query, raw_info=state.raw_info, summary=response.content
+        )
+
+    # Node 3: Format the final summary for output
+    def output_summary(state: ResearchState) -> ResearchState:
+        final_summary = f"Research summary for '{state.query}': {state.summary}"
+        return ResearchState(
+            query=state.query, raw_info=state.raw_info, summary=final_summary
+        )
+
+    # Add nodes to the graph
+    graph.add_node("search_info", search_info)
+    graph.add_node("summarize_info", summarize_info)
+    graph.add_node("output_summary", output_summary)
+
+    # Define the flow between nodes (edges)
+    graph.add_edge("search_info", "summarize_info")
+    graph.add_edge("summarize_info", "output_summary")
+
+    # Set the starting and ending points of the graph
+    graph.set_entry_point("search_info")
+    graph.set_finish_point("output_summary")
+
+    # Compile the graph into an executable app
+    return graph.compile()
+
+
+# Function to run the research agent with a given query prompt
+def run_research_agent(prompt):
+    # Get the compiled graph application
+    app = get_research_agent()
+    # Run the asynchronous invocation and get the result
+    result = asyncio.run(app.ainvoke(ResearchState(query=prompt)))
+    return result

examples/integration-langgraph/promptfooconfig.yaml

@@ -0,0 +1,35 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'LangGraph Research Agent Evaluation'
+
+# List of input prompts to test the provider with
+prompts:
+  - '{{input_prompt}}'
+
+# Provider configuration: links to the local Python provider script
+providers:
+  - id: file://./provider.py # Local provider file path
+    label: Research Agent # Label to display in results
+
+# Default test assertions applied to all tests (unless overridden)
+defaultTest:
+  assert:
+    - type: is-json # Check that output is valid JSON
+      value:
+        type: object # Ensure the output is a JSON object
+        properties: # Define expected keys and their types
+          query:
+            type: string
+          raw_info:
+            type: string
+          summary:
+            type: string
+        required: ['query', 'raw_info', 'summary'] # All keys must be present
+
+# Specific test cases
+tests:
+  - description: 'Basic research test' # Name/description of this test
+    vars:
+      input_prompt: 'latest AI research trends' # Value for input_prompt variable
+    assert:
+      - type: python # Custom Python assertion
+        value: "'summary' in output" # Check that 'summary' exists in the output

examples/integration-langgraph/provider.py

@@ -0,0 +1,32 @@
+from agent import run_research_agent
+
+
+# Main API function that external tools or systems will call
+def call_api(prompt, options, context):
+    """
+    Executes the research agent with the given prompt.
+
+    Args:
+        prompt (str): The research query or question.
+        options (dict): Additional options for future extension (currently unused).
+        context (dict): Contextual information (currently unused).
+
+    Returns:
+        dict: A dictionary containing the agent's output or an error message.
+    """
+    try:
+        # Run the research agent and get the result
+        result = run_research_agent(prompt)
+        # Wrap and return the result inside a dictionary
+        return {"output": result}
+    except Exception as e:
+        # Handle any exceptions and return an error summary
+        return {"output": {"summary": f"Error: {str(e)}"}}
+
+
+# If this file is run directly, execute a simple test
+if __name__ == "__main__":
+    print("✅ Testing Research Agent provider...")
+    test_prompt = "latest AI research trends"
+    result = call_api(test_prompt, {}, {})
+    print("Provider result:", result)