ludwig-ai--ludwig/.github/workflows/pytest.yml

name: pytest

on:
  push:
    branches: ["main", "release-*", "future-capabilities"]
  pull_request:
    branches: ["main", "release-*", "future-capabilities"]

concurrency:
  group: pytest-${{ github.head_ref || github.sha }}
  cancel-in-progress: true

jobs:
  unit-tests:
    runs-on: ubuntu-latest
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.LUDWIG_TESTS_AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.LUDWIG_TESTS_AWS_SECRET_ACCESS_KEY }}
      KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
      KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
      IS_NOT_FORK: ${{ !(github.event.pull_request.base.repo.full_name == 'ludwig-ai/ludwig' && github.event.pull_request.head.repo.fork) }}
      UV_SYSTEM_PYTHON: "1"

    name: Unit Tests
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Setup Linux
        run: |
          sudo apt-get update && sudo apt-get install -y cmake libsndfile1 libsox-dev ffmpeg

      - name: uv cache
        uses: actions/cache@v4
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-uv-unit-${{ hashFiles('pyproject.toml', '.github/workflows/pytest.yml') }}

      - name: Install uv
        run: pip install uv

      - name: Install dependencies
        run: |
          uv pip install torch==2.12.0 torchvision==0.27.0 torchaudio==2.11.0 torchcodec --extra-index-url https://download.pytorch.org/whl/cpu
          uv pip install '.[test]'
          uv pip list

      - name: Unit Tests
        run: |
          RUN_PRIVATE=$IS_NOT_FORK LUDWIG_TEST_SUITE_TIMEOUT_S=5400 pytest -v --timeout 300 --durations 100 -m "not distributed and not slow and not combinatorial and not llm" --junitxml pytest.xml tests/ludwig

      - name: Regression Tests
        run: |
          RUN_PRIVATE=$IS_NOT_FORK LUDWIG_TEST_SUITE_TIMEOUT_S=5400 pytest -v --timeout 300 --durations 100 -m "not distributed and not slow and not combinatorial and not llm" --junitxml pytest-regression.xml tests/regression_tests

      - name: Upload Test Results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: Unit Test Results
          path: pytest*.xml

  integration-tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        test-markers:
          - "integration_tests_a"
          - "integration_tests_b"
          - "integration_tests_c"
          - "integration_tests_d"
          - "integration_tests_e"
          - "integration_tests_f"
          - "integration_tests_g"
          - "integration_tests_h"
          - "integration_tests_i"
          - "integration_tests_j"

    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.LUDWIG_TESTS_AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.LUDWIG_TESTS_AWS_SECRET_ACCESS_KEY }}
      KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
      KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
      IS_NOT_FORK: ${{ !(github.event.pull_request.base.repo.full_name == 'ludwig-ai/ludwig' && github.event.pull_request.head.repo.fork) }}
      MARKERS: ${{ matrix.test-markers }}
      UV_SYSTEM_PYTHON: "1"
      MLFLOW_ALLOW_FILE_STORE: "true"

    name: Integration (${{ matrix.test-markers }})
    services:
      minio:
        image: fclairamb/minio-github-actions
        env:
          MINIO_ACCESS_KEY: minio
          MINIO_SECRET_KEY: minio123
        ports:
          - 9000:9000

    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Setup Linux
        run: |
          sudo apt-get update && sudo apt-get install -y cmake libsndfile1 libsox-dev ffmpeg

      - name: uv cache
        uses: actions/cache@v4
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-uv-integration-${{ hashFiles('pyproject.toml', '.github/workflows/pytest.yml') }}

      - name: Install dependencies
        run: |
          pip install uv
          uv pip install torch==2.12.0 torchvision==0.27.0 torchaudio==2.11.0 torchcodec --extra-index-url https://download.pytorch.org/whl/cpu
          uv pip install '.[test]'
          uv pip list

      - name: Free Disk Space
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: false
          docker-images: true
          swap-storage: true

      - name: Integration Tests
        run: |
          RUN_PRIVATE=$IS_NOT_FORK LUDWIG_TEST_SUITE_TIMEOUT_S=7200 pytest -v --timeout 300 --durations 100 -m "not slow and not combinatorial and not llm and $MARKERS" --junitxml pytest.xml tests/integration_tests

      - name: Upload Test Results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: Integration Test Results (${{ matrix.test-markers }})
          path: pytest.xml

  distributed-tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        distributed-group:
          - "distributed_a"
          - "distributed_b"
          - "distributed_c"
          - "distributed_d"
          - "distributed_e"
          - "distributed_f"

    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.LUDWIG_TESTS_AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.LUDWIG_TESTS_AWS_SECRET_ACCESS_KEY }}
      KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
      KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
      IS_NOT_FORK: ${{ !(github.event.pull_request.base.repo.full_name == 'ludwig-ai/ludwig' && github.event.pull_request.head.repo.fork) }}
      DIST_GROUP: ${{ matrix.distributed-group }}
      UV_SYSTEM_PYTHON: "1"
      MLFLOW_ALLOW_FILE_STORE: "true"

    name: Distributed (${{ matrix.distributed-group }})
    services:
      minio:
        image: fclairamb/minio-github-actions
        env:
          MINIO_ACCESS_KEY: minio
          MINIO_SECRET_KEY: minio123
        ports:
          - 9000:9000

    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Setup Linux
        run: |
          sudo apt-get update && sudo apt-get install -y cmake libsndfile1 libsox-dev ffmpeg

      - name: uv cache
        uses: actions/cache@v4
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-uv-distributed-${{ hashFiles('pyproject.toml', '.github/workflows/pytest.yml') }}

      - name: Install dependencies
        run: |
          pip install uv
          uv pip install torch==2.12.0 torchvision==0.27.0 torchaudio==2.11.0 torchcodec --extra-index-url https://download.pytorch.org/whl/cpu
          uv pip install '.[test]'
          uv pip list

      - name: Free Disk Space
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: false
          docker-images: true
          swap-storage: true

      - name: Distributed Tests
        run: |
          RUN_PRIVATE=$IS_NOT_FORK LUDWIG_TEST_SUITE_TIMEOUT_S=5400 pytest -v --timeout 300 --durations 100 -m "distributed and not slow and not combinatorial and not llm and $DIST_GROUP" --ignore=tests/integration_tests/test_server.py --junitxml pytest.xml tests/ludwig tests/integration_tests

      - name: Upload Test Results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: Distributed Test Results (${{ matrix.distributed-group }})
          path: pytest.xml

  test-minimal-install:
    name: Minimal Install
    runs-on: ubuntu-latest
    timeout-minutes: 15
    env:
      UV_SYSTEM_PYTHON: "1"
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Setup Linux
        run: |
          sudo apt-get update && sudo apt-get install -y cmake libsndfile1

      - name: Install dependencies
        run: |
          pip install uv
          uv pip install torch==2.12.0 torchvision==0.27.0 torchaudio==2.11.0 torchcodec --extra-index-url https://download.pytorch.org/whl/cpu
          uv pip install .
          uv pip list

      - name: Check Install
        run: |
          ludwig check_install

  event_file:
    name: "Event File"
    runs-on: ubuntu-latest
    steps:
      - name: Upload
        uses: actions/upload-artifact@v4
        with:
          name: Event File
          path: ${{ github.event_path }}