huggingface--datasets/.github/workflows/ci.yml

name: CI

on:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main
      - ci-*

env:
  CI_HEADERS: ${{ secrets.CI_HEADERS }}

jobs:

  check_code_quality:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
        with:
          python-version: "3.10"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install .[quality]
      - name: Check quality
        run: |
          ruff check tests src benchmarks utils setup.py # linter
          ruff format --check tests src benchmarks utils setup.py # formatter

  test:
    needs: check_code_quality
    strategy:
      matrix:
        test: ['unit', 'integration']
        os: [ubuntu-latest, windows-latest]
        deps_versions: [deps-latest, deps-minimum]
    continue-on-error: ${{ matrix.test == 'integration' }}
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          fetch-depth: 0
      - name: Setup FFmpeg
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          sudo apt update
          sudo apt install -y ffmpeg 
      - name: Set up Python 3.10
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
        with:
          python-version: "3.10"
      - name: Setup conda env (windows)
        if: ${{ matrix.os == 'windows-latest' }}
        uses: conda-incubator/setup-miniconda@9f54435e0e72c53962ee863144e47a4b094bfd35  # v2
        with:
          auto-update-conda: true
          miniconda-version: "latest"
          activate-environment: test
          python-version: "3.10"
      - name: Setup FFmpeg (windows)
        if: ${{ matrix.os == 'windows-latest' }}
        run: conda install "ffmpeg=7.0.1" -c conda-forge
      - name: Upgrade pip
        run: python -m pip install --upgrade pip
      - name: Install uv
        run: pip install --upgrade uv
      - name: Install dependencies
        run: uv pip install --system "datasets[tests] @ ."
      - name: Install dependencies (latest versions)
        if: ${{ matrix.deps_versions == 'deps-latest' }}
        run: uv pip install --system --upgrade pyarrow huggingface-hub "dill<0.3.9"
      - name: Install dependencies (minimum versions)
        if: ${{ matrix.deps_versions != 'deps-latest' }}
        run: uv pip install --system pyarrow==21.0.0 huggingface-hub==0.25.0 transformers dill==0.3.1.1
      - name: Print dependencies
        run: uv pip list
      - name: Test with pytest
        run: |
          python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/

  test_py314:
    needs: check_code_quality
    strategy:
      matrix:
        test: ['unit']
        os: [ubuntu-latest, windows-latest]
        deps_versions: [deps-latest]
    continue-on-error: false
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          fetch-depth: 0
      - name: Setup FFmpeg
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          sudo apt update
          sudo apt install -y ffmpeg
      - name: Set up Python 3.14
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
        with:
          python-version: "3.14"
      - name: Setup conda env (windows)
        if: ${{ matrix.os == 'windows-latest' }}
        uses: conda-incubator/setup-miniconda@9f54435e0e72c53962ee863144e47a4b094bfd35  # v2
        with:
          auto-update-conda: true
          miniconda-version: "latest"
          activate-environment: test
          python-version: "3.14"
      - name: Setup FFmpeg (windows)
        if: ${{ matrix.os == 'windows-latest' }}
        run: conda install "ffmpeg=7.0.1" -c conda-forge
      - name: Upgrade pip
        run: python -m pip install --upgrade pip
      - name: Install uv
        run: pip install --upgrade uv
      - name: Install dependencies
        run: uv pip install --system "datasets[tests] @ ."
      - name: Install tsfile (py3.14 only)
        run: uv pip install --system "tsfile>=2.3.0"
      - name: Print dependencies
        run: uv pip list
      - name: Test with pytest
        run: |
          python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/

  test_py314_future:
    needs: check_code_quality
    strategy:
      matrix:
        test: ['unit']
        os: [ubuntu-latest, windows-latest]
        deps_versions: [deps-latest]
    continue-on-error: false
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          fetch-depth: 0
      - name: Setup FFmpeg
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          sudo apt update
          sudo apt install -y ffmpeg 
      - name: Set up Python 3.14
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
        with:
          python-version: "3.14"
      - name: Setup conda env (windows)
        if: ${{ matrix.os == 'windows-latest' }}
        uses: conda-incubator/setup-miniconda@9f54435e0e72c53962ee863144e47a4b094bfd35  # v2
        with:
          auto-update-conda: true
          miniconda-version: "latest"
          activate-environment: test
          python-version: "3.14"
      - name: Setup FFmpeg (windows)
        if: ${{ matrix.os == 'windows-latest' }}
        run: conda install "ffmpeg=7.0.1" -c conda-forge
      - name: Upgrade pip
        run: python -m pip install --upgrade pip
      - name: Install uv
        run: pip install --upgrade uv
      - name: Install dependencies
        run: uv pip install --system "datasets[tests_numpy2] @ ."
      - name: Print dependencies
        run: pip list

      - name: Test with pytest
        run: |
          python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/