unstructured-io--unstructured/.github/workflows/ci.yml

name: CI

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
  merge_group:
    branches: [ main ]

permissions:
  id-token: write
  contents: read

jobs:
  setup:
    strategy:
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: ./.github/actions/base-cache
      with:
        python-version: ${{ matrix.python-version }}

  check-licenses:
    strategy:
      matrix:
        python-version: [ "3.12" ]
    runs-on: ubuntu-latest
    needs: [setup]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/base-cache
        with:
          python-version: ${{ matrix.python-version }}
      - name: Check licenses
        run: make check-licenses

  lint:
    strategy:
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
    runs-on: ubuntu-latest
    needs: [setup, changelog]
    steps:
    - uses: actions/checkout@v4
    - uses: ./.github/actions/base-cache
      with:
        python-version: ${{ matrix.python-version }}
    - name: Lint
      run: make check

  shellcheck:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: ShellCheck
        uses: ludeeus/action-shellcheck@master

  shfmt:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: setup shfmt
        uses: mfinelli/setup-shfmt@v3
      - name: Run shfmt
        run: shfmt -i 2 -d .


  test_unit:
    strategy:
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
    runs-on: opensource-linux-8core
    needs: [setup, lint]
    steps:
    - uses: actions/checkout@v4
    - uses: ./.github/actions/base-cache
      with:
        python-version: ${{ matrix.python-version }}
    - name: Test
      env:
        UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
        TESSERACT_VERSION: "5.5.1"
      run: |
        retry() {
          local n=1 max=5 delay=15
          while true; do
            "$@" && return 0
            if (( n >= max )); then
              echo "Command failed after $n attempts: $*" >&2
              return 1
            fi
            echo "Attempt $n/$max failed for: $*. Retrying in ${delay}s..." >&2
            sleep "$delay"
            n=$((n+1))
          done
        }
        retry sudo apt-get update
        retry sudo apt-get install -y libmagic-dev poppler-utils libreoffice
        retry sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
        retry sudo apt-get update -o "APT::Update::Error-Mode=any"
        retry sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
        tesseract --version
        installed_tesseract_version=$(tesseract --version | grep -oP '(?<=tesseract )\d+\.\d+\.\d+')
        if [ "$installed_tesseract_version" != "${{env.TESSERACT_VERSION}}" ]; then
          echo "Tesseract version ${{env.TESSERACT_VERSION}} is required but found version $installed_tesseract_version"
          exit 1
        fi
        make test CI=true UNSTRUCTURED_INCLUDE_DEBUG_METADATA=true
        make check-coverage

  test_unit_no_extras:
    strategy:
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
    runs-on: ubuntu-latest
    needs: [setup, lint]
    steps:
    - uses: actions/checkout@v4
    - name: Install uv
      uses: astral-sh/setup-uv@v5
      with:
        enable-cache: true
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install base dependencies only (no extras)
      env:
        UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
      run: |
        uv sync --locked --group test
        make test-no-extras CI=true

  test_unit_dependency_extras:
    strategy:
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
        extra: ["csv", "docx", "odt", "markdown", "pypandoc", "pdf-image", "pptx", "xlsx"]
        include:
          - extra: csv
            uv-extras: "--extra csv"
          - extra: docx
            uv-extras: "--extra docx"
          - extra: odt
            uv-extras: "--extra odt"
          - extra: markdown
            uv-extras: "--extra md"
          - extra: pypandoc
            uv-extras: "--extra epub --extra org --extra rtf --extra rst"
          - extra: pdf-image
            uv-extras: "--extra pdf --extra image --extra paddleocr"
          - extra: pptx
            uv-extras: "--extra pptx"
          - extra: xlsx
            uv-extras: "--extra xlsx"
    runs-on: opensource-linux-8core
    needs: [setup, lint, test_unit_no_extras]
    steps:
    - uses: actions/checkout@v4
    - name: Install uv
      uses: astral-sh/setup-uv@v5
      with:
        enable-cache: true
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install extra dependencies
      run: |
        uv sync --locked ${{ matrix.uv-extras }} --group test
    - name: Install system dependencies
      run: |
        retry() {
          local n=1 max=5 delay=15
          while true; do
            "$@" && return 0
            if (( n >= max )); then
              echo "Command failed after $n attempts: $*" >&2
              return 1
            fi
            echo "Attempt $n/$max failed for: $*. Retrying in ${delay}s..." >&2
            sleep "$delay"
            n=$((n+1))
          done
        }
        retry sudo apt-get update
        retry sudo apt-get install -y libmagic-dev poppler-utils libreoffice
        retry sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
        retry sudo apt-get update -o "APT::Update::Error-Mode=any"
        retry sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
        tesseract --version
    - name: Test
      env:
        UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
        UNSTRUCTURED_HF_TOKEN: ${{ secrets.HF_TOKEN }}
      run: |
        make test-extra-${{ matrix.extra }} CI=true

  test_ingest_src:
    strategy:
      matrix:
        python-version: ["3.12"]
    runs-on: opensource-linux-8core
    needs: [setup, lint]
    steps:
    # actions/checkout MUST come before auth
    - uses: 'actions/checkout@v4'
    - uses: ./.github/actions/base-cache
      with:
        python-version: ${{ matrix.python-version }}
    - name: Set up Docker
      uses: docker/setup-docker-action@v4
    - name: Set up Docker Compose
      uses: docker/setup-compose-action@v1
    - name: Test (end-to-end)
      env:
        AIRTABLE_PERSONAL_ACCESS_TOKEN: ${{ secrets.AIRTABLE_PERSONAL_ACCESS_TOKEN }}
        BOX_APP_CONFIG: ${{ secrets.BOX_APP_CONFIG }}
        CONFLUENCE_API_TOKEN: ${{ secrets.CONFLUENCE_API_TOKEN }}
        CONFLUENCE_USER_EMAIL: ${{ secrets.CONFLUENCE_USER_EMAIL }}
        DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
        DROPBOX_APP_KEY: ${{ secrets.DROPBOX_APP_KEY }}
        DROPBOX_APP_SECRET: ${{ secrets.DROPBOX_APP_SECRET }}
        DROPBOX_REFRESH_TOKEN: ${{ secrets.DROPBOX_REFRESH_TOKEN }}
        GCP_INGEST_SERVICE_KEY: ${{ secrets.GCP_INGEST_SERVICE_KEY }}
        GH_READ_ONLY_ACCESS_TOKEN: ${{ secrets.GH_READ_ONLY_ACCESS_TOKEN }}
        HUBSPOT_API_TOKEN: ${{ secrets.HUBSPOT_API_TOKEN }}
        JIRA_INGEST_API_TOKEN: ${{ secrets.JIRA_INGEST_API_TOKEN }}
        JIRA_INGEST_USER_EMAIL: ${{ secrets.JIRA_INGEST_USER_EMAIL }}
        MONGODB_URI: ${{ secrets.MONGODB_URI }}
        MONGODB_DATABASE_NAME: ${{ secrets.MONGODB_DATABASE_NAME }}
        MS_CLIENT_CRED: ${{ secrets.MS_CLIENT_CRED }}
        MS_CLIENT_ID: ${{ secrets.MS_CLIENT_ID }}
        MS_TENANT_ID: ${{ secrets.MS_TENANT_ID }}
        MS_USER_EMAIL: ${{ secrets.MS_USER_EMAIL }}
        MS_USER_PNAME: ${{ secrets.MS_USER_PNAME }}
        SALESFORCE_USERNAME: ${{secrets.SALESFORCE_USERNAME}}
        SALESFORCE_CONSUMER_KEY: ${{secrets.SALESFORCE_CONSUMER_KEY}}
        SALESFORCE_PRIVATE_KEY: ${{secrets.SALESFORCE_PRIVATE_KEY}}
        SHAREPOINT_CLIENT_ID: ${{secrets.SHAREPOINT_CLIENT_ID}}
        SHAREPOINT_CRED: ${{secrets.SHAREPOINT_CRED}}
        SHAREPOINT_SITE: ${{secrets.SHAREPOINT_SITE}}
        SHAREPOINT_PERMISSIONS_APP_ID: ${{secrets.SHAREPOINT_PERMISSIONS_APP_ID}}
        SHAREPOINT_PERMISSIONS_APP_CRED: ${{secrets.SHAREPOINT_PERMISSIONS_APP_CRED}}
        SHAREPOINT_PERMISSIONS_TENANT: ${{secrets.SHAREPOINT_PERMISSIONS_TENANT}}
        SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
        UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
        NOTION_API_KEY: ${{ secrets.NOTION_API_KEY }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }}
        AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }}
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        OCTOAI_API_KEY: ${{ secrets.OCTOAI_API_KEY }}
        PINECONE_API_KEY: ${{secrets.PINECONE_API_KEY}}
        ASTRA_DB_APPLICATION_TOKEN: ${{secrets.ASTRA_DB_TOKEN}}
        ASTRA_DB_API_ENDPOINT: ${{secrets.ASTRA_DB_ENDPOINT}}
        MXBAI_API_KEY: ${{secrets.MXBAI_API_KEY}}
        OCR_AGENT: "unstructured.partition.utils.ocr_models.tesseract_ocr.OCRAgentTesseract"
        CI: "true"
      run: |
        retry() {
          local n=1 max=5 delay=15
          while true; do
            "$@" && return 0
            if (( n >= max )); then
              echo "Command failed after $n attempts: $*" >&2
              return 1
            fi
            echo "Attempt $n/$max failed for: $*. Retrying in ${delay}s..." >&2
            sleep "$delay"
            n=$((n+1))
          done
        }
        retry sudo apt-get update
        retry sudo apt-get install -y libmagic-dev poppler-utils libreoffice
        retry sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
        retry sudo apt-get update -o "APT::Update::Error-Mode=any"
        retry sudo apt-get install -y tesseract-ocr
        retry sudo apt-get install -y tesseract-ocr-kor
        retry sudo apt-get install -y diffstat
        tesseract --version
        uv run --no-sync ./test_unstructured_ingest/test-ingest-src.sh

  test_json_to_html:
    strategy:
      matrix:
        python-version: ["3.12"]
    runs-on: ubuntu-latest
    needs: [setup, lint]
    steps:
    - uses: 'actions/checkout@v4'
    - uses: ./.github/actions/base-cache
      with:
        python-version: ${{ matrix.python-version }}
    - name: Test HTML fixtures
      env:
        OVERWRITE_FIXTURES: "false"
      run: |
        sudo apt-get install diffstat
        uv run --no-sync ./test_unstructured_ingest/check-diff-expected-output-html.sh

  test_json_to_markdown:
    strategy:
      matrix:
        python-version: ["3.12"]
    runs-on: ubuntu-latest
    needs: [setup, lint]
    steps:
    - uses: 'actions/checkout@v4'
    - uses: ./.github/actions/base-cache
      with:
        python-version: ${{ matrix.python-version }}
    - name: Test markdown fixtures
      env:
        OVERWRITE_FIXTURES: "false"
      run: |
        sudo apt-get install diffstat
        uv run --no-sync ./test_unstructured_ingest/check-diff-expected-output-markdown.sh

  changelog:
    runs-on: ubuntu-latest
    steps:
    # need to checkout otherwise paths-filter will fail on merge-queue trigger
    - uses: actions/checkout@v4
    - if: github.ref != 'refs/heads/main'
      uses: dorny/paths-filter@v3
      id: changes
      with:
        filters: |
          src:
            - 'unstructured/**'

    - if: steps.changes.outputs.src == 'true' && github.ref != 'refs/heads/main'
      uses: dangoslen/changelog-enforcer@v3

  # TODO - figure out best practice for caching docker images
  # (Using the virtualenv to get pytest)
  test_dockerfile:
    runs-on: opensource-linux-8core
    needs: [ setup, lint ]
    steps:
      - uses: actions/checkout@v4
      - name: Test Dockerfile
        run: |
          echo "UNS_API_KEY=${{ secrets.UNS_API_KEY }}" > uns_test_env_file
          make docker-build
          make docker-test CI=true UNSTRUCTURED_INCLUDE_DEBUG_METADATA=true
      - name: Scan image
        uses: anchore/scan-action@v3
        with:
          image: "unstructured:dev"
          severity-cutoff: critical
          fail-build: false
          only-fixed: true
          output-format: table