name: tests

on:
  push:
    branches: [main]
    paths-ignore:
      - 'docs/**'
      - 'method_comparison/**'
      - '**.md'
  pull_request:
    paths-ignore:
      - 'docs/**'
      - 'method_comparison/**'
      - '**.md'

env:
  HF_HOME: .cache/huggingface

permissions: {}

jobs:
  check_code_quality:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          persist-credentials: false
      - name: Set up Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
        with:
          python-version: "3.11"
          cache: "pip"
          cache-dependency-path: "setup.py"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install .[dev]
      - name: Check quality
        run: |
          make quality

  tests:
    needs: check_code_quality
    # dependabot updates (which don't require approval for CI to run) shouldn't trigger unit tests
    if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.user.login == 'dependabot[bot]') }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13"]
        os: ["ubuntu-latest", "windows-latest"]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          persist-credentials: false
      - name: Make space for cache + models
        # Ubuntu runner have less space free which is problematic since the model
        # cache + dependencies fill up the disk, leaving no space for execution.
        # So we remove some of the stuff we don't need (Java, .NET, etc.)
        #
        # Idea: https://dev.to/mathio/squeezing-disk-space-from-github-actions-runners-an-engineers-guide-3pjg
        if: matrix.os != 'windows-latest'
        run: |
          df -h

          # Remove Java (JDKs)
          sudo rm -rf /usr/lib/jvm

          # Remove .NET SDKs
          sudo rm -rf /usr/share/dotnet

          # Remove Swift toolchain
          sudo rm -rf /usr/share/swift

          # Remove Haskell (GHC)
          sudo rm -rf /usr/local/.ghcup

          # Remove Julia
          sudo rm -rf /usr/local/julia*

          # Remove Android SDKs
          sudo rm -rf /usr/local/lib/android

          # Remove Chromium (optional if not using for browser tests)
          sudo rm -rf /usr/local/share/chromium

          # Remove Microsoft/Edge and Google Chrome builds
          sudo rm -rf /opt/microsoft /opt/google

          # Remove Azure CLI
          sudo rm -rf /opt/az

          # Remove PowerShell
          sudo rm -rf /usr/local/share/powershell

          # Remove CodeQL and other toolcaches
          sudo rm -rf /opt/hostedtoolcache

          df -h
      - name: Model cache
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        with:
          # Avoid caching HF_HOME/modules and Python cache files to prevent interoperability
          # issues and potential cache poisioning. We also avoid lock files to prevent runs
          # avoiding re-download because they see a lock file.
          path: |
            ${{ env.HF_HOME }}/hub/**
            !${{ env.HF_HOME }}/**/*.pyc
          key: model-cache-${{ github.run_id }}
          restore-keys: model-cache-
          enableCrossOsArchive: true

      - name: Dump cache content
        # TODO: remove this step after 2025-02-15
        if: matrix.os != 'windows-latest'
        run: |
          SHASUM=sha256sum
          [ -f "$(which shasum)" ] && SHASUM=shasum
          find "${{ env.HF_HOME }}/hub" -type f -exec "$SHASUM" {} \; > cache_content_initial || true
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: "setup.py"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install setuptools
          # cpu version of pytorch
          pip install -e .[test]
      - name: Test with pytest
        shell: bash
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          TRANSFORMERS_IS_CI: 1
          CI: 1
        run: |
          make test
          # clean up all pytest temporary directories that are kept due to retention since space
          # is a scarce resource on the runners and tasks like model cache creation (further below)
          # fail if there's not enough space available.
          (rm -r "/tmp/pytest-of-$(id -u -n)" || true)
      - name: Dump cache content and diff
        # This is just debug info so that we can monitor if the model cache diverges substantially
        # over time and what the diverging model is.
        # TODO: remove after 2025-02-15
        if: matrix.os != 'windows-latest'
        run: |
          SHASUM=sha256sum
          [ -f "$(which shasum)" ] && SHASUM=shasum
          find "${{ env.HF_HOME }}/hub" -type f -exec "$SHASUM" {} \; > cache_content_after || true
          diff -udp cache_content_initial cache_content_after || true
      - name: Delete old model cache entries
        run: |
          # make sure that cache cleaning doesn't break the pipeline
          python scripts/ci_clean_cache.py -d || true
      - name: Update model cache
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        # Only let one runner (preferably the one that covers most tests) update the model cache
        # after *every* run. This way we make sure that our cache is never outdated and we don't
        # have to keep track of hashes.
        if: always() && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10'
        with:
          path: |
            ${{ env.HF_HOME }}/hub/**
            !${{ env.HF_HOME }}/**/*.pyc
          key: model-cache-${{ github.run_id }}