name: Core Tests

on:
  push:
    paths-ignore:
      - 'typescript/**'
  pull_request:
    paths-ignore:
      - 'typescript/**'
  workflow_dispatch:

jobs:
  test:
    runs-on: ubuntu-latest
    env:
      # Expose once at job level because forked PRs can't use secrets.* in `if:` conditions.
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      DEEPEVAL_TELEMETRY_OPT_OUT: 1
    steps:
      #----------------------------------------------
      #       check-out repo and set-up python
      #----------------------------------------------
      - name: Check out repository
        uses: actions/checkout@v3
      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      #----------------------------------------------
      #  -----  install & configure poetry  -----
      #----------------------------------------------
      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      #----------------------------------------------
      #       load cached venv if cache exists
      #----------------------------------------------
      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
      #----------------------------------------------
      # install dependencies if cache does not exist
      #----------------------------------------------
      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root --only main
      #----------------------------------------------
      # install your root project, if required
      #----------------------------------------------

      - name: Install project
        run: poetry install --no-interaction --only main

      #----------------------------------------------
      #    install dev dependencies (including chromadb and model deps)
      #----------------------------------------------
      - name: Install dev dependencies
        run: poetry install --no-interaction --with dev
        
      #----------------------------------------------
      #              run test suite
      #----------------------------------------------

      # Run tests (with secrets): full suite
      - name: Run tests
        if: ${{ env.OPENAI_API_KEY != '' }}
        run: |
          poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys \
            tests/test_core/ \
            --ignore=tests/test_core/test_synthesizer/ \
            --ignore=tests/test_core/test_datasets/

      # Run tests (no secrets): skip e2e that require API keys
      - name: Run tests (no secrets)
        if: ${{ env.OPENAI_API_KEY == '' }}
        run: |
          poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys tests/test_core/  \
          --ignore=tests/test_core/test_synthesizer/                                \
          --ignore=tests/test_core/test_datasets/                                   \
          --ignore=tests/test_core/test_tracing/test_dataset_iterator.py            \
          --ignore=tests/test_core/test_evaluation/test_end_to_end/test_configs.py

      # Dev tests (with secrets)
      - name: Run dev tests
        if: ${{ env.OPENAI_API_KEY != '' }}
        run: |
          poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys -o faulthandler_timeout=300 \
          tests/test_core/test_synthesizer/ tests/test_core/test_datasets/ tests/test_core/test_simulator/

      # Dev tests (no secrets)
      - name: Run dev tests (no secrets)
        if: ${{ env.OPENAI_API_KEY == '' }}
        run: |
          poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys tests/test_core/test_synthesizer/ tests/test_core/test_datasets/ tests/test_core/test_simulator/ \
          --ignore=tests/test_core/test_tracing/test_dataset_iterator.py            \
          --ignore=tests/test_core/test_synthesizer/test_context_generator.py       \
          --ignore=tests/test_core/test_simulator/test_conversation_simulator.py    \
          --ignore=tests/test_core/test_synthesizer/test_generate_from_goldens.py   \
          --ignore=tests/test_core/test_synthesizer/test_synthesizer.py