confident-ai--deepeval/.github/workflows/test_integrations.yml

name: All Integration Tests

on:
  push:
    paths-ignore:
      - 'typescript/**'
  pull_request:
    paths-ignore:
      - 'typescript/**'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # ===========================================================================
  # 1. LangChain / LangGraph Tests
  # ===========================================================================
  langchain:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      DEEPEVAL_TELEMETRY_OPT_OUT: "1"
      PYTHONUNBUFFERED: "1"
      PYTEST_ADDOPTS: "-vv -rA --maxfail=1 --capture=tee-sys"

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Install Dependencies
        run: |
          poetry install --no-interaction --no-root --only main
          poetry run pip install -U langgraph langchain langchain-openai

      - name: Install Project
        run: poetry install --no-interaction --only main

      - name: Run LangChain/LangGraph Tests
        if: ${{ env.OPENAI_API_KEY != '' }}
        run: |
          poetry run pytest tests/test_integrations/test_langchain/
          poetry run pytest tests/test_integrations/test_langgraph/

  # ===========================================================================
  # 2. CrewAI Tests
  # ===========================================================================
  crewai:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      DEEPEVAL_TELEMETRY_OPT_OUT: "1"
      PYTHONUNBUFFERED: "1"

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Install Dependencies
        run: |
          poetry install --no-interaction --no-root --only main
          poetry install --with integrations
          poetry run pip install -U crewai
          # Pin to pydantic-ai-slim[openai] to avoid the meta-package's
          # mistral extra (mistralai is currently quarantined on PyPI;
          # see comment in pyproject.toml integrations group).
          poetry run pip install -U "pydantic-ai-slim[openai]"

      - name: Install Project
        run: poetry install --no-interaction --only main

      - name: Run CrewAI Tests
        if: ${{ env.OPENAI_API_KEY != '' }}
        run: poetry run pytest tests/test_integrations/test_crewai/ 

  # ===========================================================================
  # 3. Pydantic AI Tests
  # ===========================================================================
  pydantic-ai:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      DEEPEVAL_TELEMETRY_OPT_OUT: "1"
      PYTHONUNBUFFERED: "1"

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Install Dependencies (Pydantic AI)
        run: |
          poetry install --no-interaction --no-root --only main
          poetry install --with integrations
          # Pin to pydantic-ai-slim[openai] to avoid the meta-package's
          # mistral extra (mistralai is currently quarantined on PyPI;
          # see comment in pyproject.toml integrations group).
          poetry run pip install -U "pydantic-ai-slim[openai]"

      - name: Run Pydantic AI Tests
        if: ${{ env.OPENAI_API_KEY != '' }}
        run: poetry run pytest tests/test_integrations/test_pydanticai/

  # ===========================================================================
  # 4. LlamaIndex Tests
  # ===========================================================================
  llama-index:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      DEEPEVAL_TELEMETRY_OPT_OUT: "1"
      PYTHONUNBUFFERED: "1"

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Install Dependencies (LlamaIndex)
        run: |
          poetry install --no-interaction --no-root --only main
          poetry install --with integrations
          poetry run pip install -U llama-index

      - name: Run LlamaIndex Tests
        if: ${{ env.OPENAI_API_KEY != '' }}
        run: poetry run pytest tests/test_integrations/test_llamaindex/

  # ===========================================================================
  # 5. OpenAI Agents Tests
  # ===========================================================================
  openai-agents:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      DEEPEVAL_TELEMETRY_OPT_OUT: "1"
      PYTHONUNBUFFERED: "1"

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Install Dependencies (OpenAI Agents)
        run: |
          poetry install --no-interaction --no-root --only main
          poetry install --with integrations
          poetry run pip install -U openai-agents

      - name: Run OpenAI Agents Tests
        if: ${{ env.OPENAI_API_KEY != '' }}
        run: poetry run pytest tests/test_integrations/test_openai_agents/
        
  # ===========================================================================
  # 5. OpenAI Tests
  # ===========================================================================
  openai:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      DEEPEVAL_TELEMETRY_OPT_OUT: "1"
      PYTHONUNBUFFERED: "1"

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Install Dependencies (OpenAI)
        run: |
          poetry install --no-interaction --no-root --only main
          poetry install --with integrations
          poetry run pip install -U openai

      - name: Run OpenAI Tests
        if: ${{ env.OPENAI_API_KEY != '' }}
        run: poetry run pytest tests/test_integrations/test_openai/

  # ===========================================================================
  # 6. AgentCore Tests
  # ===========================================================================
  agentcore:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      DEEPEVAL_TELEMETRY_OPT_OUT: "1"
      PYTHONUNBUFFERED: "1"

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Install Dependencies
        run: |
          poetry install --no-interaction --no-root --only main
          poetry install --with integrations
          poetry run pip install -U bedrock-agentcore strands-agents strands-agents-tools

      - name: Install Project
        run: poetry install --no-interaction --only main

      - name: Run AgentCore Tests
        if: ${{ env.OPENAI_API_KEY != '' }}
        run: |
          poetry run pytest tests/test_integrations/test_agentcore/test_async.py
          poetry run pytest tests/test_integrations/test_agentcore/test_sync.py

  # ===========================================================================
  # 7. Strands Tests
  # ===========================================================================
  strands:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      DEEPEVAL_TELEMETRY_OPT_OUT: "1"
      PYTHONUNBUFFERED: "1"

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Install Dependencies
        run: |
          poetry install --no-interaction --no-root --only main
          poetry install --with integrations
          poetry run pip install -U strands-agents

      - name: Install Project
        run: poetry install --no-interaction --only main

      - name: Run Strands Tests
        if: ${{ env.OPENAI_API_KEY != '' }}
        run: |
          poetry run pytest tests/test_integrations/test_strands/