chore: import upstream snapshot with attribution

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: bug/<short-name>
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Provide a code snippet that reproduces the issue.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment Info**
+Please run `python scripts/collect_env.py` and paste the output here. 
+This will help us understand more about the environment in which the bug occurred.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/custom.md

@@ -0,0 +1,10 @@
+---
+name: Custom issue template
+about: Describe this issue template's purpose here.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: feat/<short-name>
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/actions/base-cache/action.yml

@@ -0,0 +1,21 @@
+name: 'Base Cache Build'
+description: 'Set up uv and install project dependencies'
+inputs:
+  python-version:
+    description: 'python version associated with the environment'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: true
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+    - name: Install dependencies
+      shell: bash
+      run: |
+        uv sync --locked --all-extras --all-groups

.github/workflows/build-opencv-wheels.yml

@@ -0,0 +1,151 @@
+name: Build OpenCV Wheels (No FFmpeg)
+
+# Produces opencv-contrib-python-headless wheels compiled from source with
+# WITH_FFMPEG=OFF to eliminate bundled ffmpeg CVEs from PyPI wheels.
+# The contrib-headless variant is a strict superset of opencv-python,
+# opencv-python-headless, and opencv-contrib-python: same cv2 module with
+# core + contrib modules and no GUI / no X11. One wheel can therefore
+# satisfy all four opencv-* package names in downstream Dockerfiles.
+# Wheels are uploaded as a GitHub release so both unstructured and
+# unstructured-api Dockerfiles can download them at build time.
+
+on:
+  workflow_dispatch:
+    inputs:
+      opencv_version:
+        description: "opencv-contrib-python-headless version to build (must have sdist on PyPI)"
+        required: true
+        default: "4.12.0.88"
+      release_tag:
+        description: "GitHub release tag for the wheels (e.g. opencv-4.12.0.88)"
+        required: true
+        default: "opencv-4.12.0.88"
+
+jobs:
+  build-wheel:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runs-on: ubuntu-latest-8-cores
+            docker-platform: linux/amd64
+          - arch: arm64
+            runs-on: ubuntu-latest-arm-8-cores
+            docker-platform: linux/arm64
+    runs-on: ${{ matrix.runs-on }}
+    env:
+      OPENCV_VERSION: ${{ inputs.opencv_version }}
+      DOCKER_PLATFORM: ${{ matrix.docker-platform }}
+    steps:
+      # Docker is preinstalled on amd64 ubuntu-latest runners but not on
+      # the arm64 ubuntu-latest-arm-X-cores image. Install it on demand
+      # and make the socket accessible to the runner user without needing
+      # to re-login for the docker group to take effect.
+      - name: Install Docker (arm64 runner)
+        if: matrix.arch == 'arm64'
+        run: |
+          set -euo pipefail
+          curl -fsSL https://get.docker.com | sudo sh
+          sudo systemctl start docker
+          sudo chmod 666 /var/run/docker.sock
+          docker version --format '{{.Server.Version}}'
+
+      - name: Build opencv-contrib-python-headless from source
+        run: |
+          mkdir -p wheels
+          docker run --rm \
+            --platform="$DOCKER_PLATFORM" \
+            -e "OPENCV_VERSION=$OPENCV_VERSION" \
+            -v "$PWD/wheels:/out" \
+            cgr.dev/chainguard/wolfi-base:latest sh -c '
+              set -euo pipefail
+              # Retry apk install: the chainguard mirror occasionally returns
+              # transient errors mid-install (matches the pattern in unstructured/Dockerfile).
+              apk_ok=false
+              for attempt in 1 2 3; do
+                apk update && \
+                apk add python-3.12 python-3.12-dev python-3.12-base-dev \
+                  opencv-dev cmake gcc glibc-dev libstdc++-dev make pkgconf \
+                  py3.12-pip py3.12-numpy && \
+                apk_ok=true && break
+                echo "apk install failed (attempt $attempt/3), retrying in 5s..."
+                sleep 5
+              done
+              $apk_ok || { echo "apk install failed after 3 attempts"; exit 1; }
+
+              CMAKE_ARGS="-DWITH_FFMPEG=OFF" \
+              ENABLE_CONTRIB=1 \
+              ENABLE_HEADLESS=1 \
+              python3.12 -m pip wheel \
+                --no-binary opencv-contrib-python-headless \
+                --no-deps \
+                "opencv-contrib-python-headless==${OPENCV_VERSION}" \
+                -w /out
+
+              echo "=== Validate no bundled ffmpeg and contrib modules present ==="
+              python3.12 -m pip install /out/opencv_contrib_python_headless-*.whl
+              python3.12 -c "
+          import cv2, pathlib
+          d = pathlib.Path(cv2.__file__).parent
+          libs = d / \".libs\"
+          assert not libs.exists(), f\"Unexpected .libs dir: {list(libs.iterdir())}\"
+          # Sanity check that contrib modules made it into the build
+          assert hasattr(cv2, \"ximgproc\"), \"contrib module cv2.ximgproc missing\"
+          assert hasattr(cv2, \"aruco\"), \"contrib module cv2.aruco missing\"
+          print(f\"OK: cv2 {cv2.__version__}, contrib modules present, no bundled ffmpeg\")
+          "
+            '
+          ls -lh wheels/
+
+      - name: Upload wheel artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: opencv-wheel-${{ matrix.arch }}
+          path: wheels/opencv_contrib_python_headless-*.whl
+          retention-days: 90
+
+  create-release:
+    needs: build-wheel
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      OPENCV_VERSION: ${{ inputs.opencv_version }}
+      RELEASE_TAG: ${{ inputs.release_tag }}
+    steps:
+      - name: Download all wheel artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: wheels
+          merge-multiple: true
+
+      - name: List wheels
+        run: ls -lh wheels/
+
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # --latest=false keeps this auxiliary wheel release from displacing
+          # the actual package release on the repo's Releases page.
+          gh release create "$RELEASE_TAG" \
+            --repo "$GITHUB_REPOSITORY" \
+            --title "OpenCV Wheels ${OPENCV_VERSION} (no ffmpeg)" \
+            --latest=false \
+            --notes "$(cat <<NOTES
+          OpenCV Python contrib-headless wheels built from source with WITH_FFMPEG=OFF.
+
+          These wheels eliminate bundled ffmpeg CVEs present in the stock PyPI wheels.
+          Built against cgr.dev/chainguard/wolfi-base:latest with Python 3.12.
+
+          The contrib-headless variant provides the full cv2 API (core + contrib
+          modules, no GUI), so a single wheel can satisfy opencv-python,
+          opencv-python-headless, opencv-contrib-python, and
+          opencv-contrib-python-headless in downstream Dockerfiles.
+
+          **Source version:** opencv-contrib-python-headless==${OPENCV_VERSION}
+          **Build flags:** CMAKE_ARGS='-DWITH_FFMPEG=OFF' ENABLE_CONTRIB=1 ENABLE_HEADLESS=1
+          NOTES
+          )" \
+            wheels/*.whl

.github/workflows/ci.yml

@@ -0,0 +1,360 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  merge_group:
+    branches: [ main ]
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  setup:
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ./.github/actions/base-cache
+      with:
+        python-version: ${{ matrix.python-version }}
+
+  check-licenses:
+    strategy:
+      matrix:
+        python-version: [ "3.12" ]
+    runs-on: ubuntu-latest
+    needs: [setup]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/base-cache
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Check licenses
+        run: make check-licenses
+
+  lint:
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    runs-on: ubuntu-latest
+    needs: [setup, changelog]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ./.github/actions/base-cache
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Lint
+      run: make check
+
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: ShellCheck
+        uses: ludeeus/action-shellcheck@master
+
+  shfmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: setup shfmt
+        uses: mfinelli/setup-shfmt@v3
+      - name: Run shfmt
+        run: shfmt -i 2 -d .
+
+
+  test_unit:
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    runs-on: opensource-linux-8core
+    needs: [setup, lint]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ./.github/actions/base-cache
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Test
+      env:
+        UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
+        TESSERACT_VERSION: "5.5.1"
+      run: |
+        retry() {
+          local n=1 max=5 delay=15
+          while true; do
+            "$@" && return 0
+            if (( n >= max )); then
+              echo "Command failed after $n attempts: $*" >&2
+              return 1
+            fi
+            echo "Attempt $n/$max failed for: $*. Retrying in ${delay}s..." >&2
+            sleep "$delay"
+            n=$((n+1))
+          done
+        }
+        retry sudo apt-get update
+        retry sudo apt-get install -y libmagic-dev poppler-utils libreoffice
+        retry sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
+        retry sudo apt-get update -o "APT::Update::Error-Mode=any"
+        retry sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
+        tesseract --version
+        installed_tesseract_version=$(tesseract --version | grep -oP '(?<=tesseract )\d+\.\d+\.\d+')
+        if [ "$installed_tesseract_version" != "${{env.TESSERACT_VERSION}}" ]; then
+          echo "Tesseract version ${{env.TESSERACT_VERSION}} is required but found version $installed_tesseract_version"
+          exit 1
+        fi
+        make test CI=true UNSTRUCTURED_INCLUDE_DEBUG_METADATA=true
+        make check-coverage
+
+  test_unit_no_extras:
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    runs-on: ubuntu-latest
+    needs: [setup, lint]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: true
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install base dependencies only (no extras)
+      env:
+        UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
+      run: |
+        uv sync --locked --group test
+        make test-no-extras CI=true
+
+  test_unit_dependency_extras:
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+        extra: ["csv", "docx", "odt", "markdown", "pypandoc", "pdf-image", "pptx", "xlsx"]
+        include:
+          - extra: csv
+            uv-extras: "--extra csv"
+          - extra: docx
+            uv-extras: "--extra docx"
+          - extra: odt
+            uv-extras: "--extra odt"
+          - extra: markdown
+            uv-extras: "--extra md"
+          - extra: pypandoc
+            uv-extras: "--extra epub --extra org --extra rtf --extra rst"
+          - extra: pdf-image
+            uv-extras: "--extra pdf --extra image --extra paddleocr"
+          - extra: pptx
+            uv-extras: "--extra pptx"
+          - extra: xlsx
+            uv-extras: "--extra xlsx"
+    runs-on: opensource-linux-8core
+    needs: [setup, lint, test_unit_no_extras]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: true
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install extra dependencies
+      run: |
+        uv sync --locked ${{ matrix.uv-extras }} --group test
+    - name: Install system dependencies
+      run: |
+        retry() {
+          local n=1 max=5 delay=15
+          while true; do
+            "$@" && return 0
+            if (( n >= max )); then
+              echo "Command failed after $n attempts: $*" >&2
+              return 1
+            fi
+            echo "Attempt $n/$max failed for: $*. Retrying in ${delay}s..." >&2
+            sleep "$delay"
+            n=$((n+1))
+          done
+        }
+        retry sudo apt-get update
+        retry sudo apt-get install -y libmagic-dev poppler-utils libreoffice
+        retry sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
+        retry sudo apt-get update -o "APT::Update::Error-Mode=any"
+        retry sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
+        tesseract --version
+    - name: Test
+      env:
+        UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
+        UNSTRUCTURED_HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      run: |
+        make test-extra-${{ matrix.extra }} CI=true
+
+  test_ingest_src:
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    runs-on: opensource-linux-8core
+    needs: [setup, lint]
+    steps:
+    # actions/checkout MUST come before auth
+    - uses: 'actions/checkout@v4'
+    - uses: ./.github/actions/base-cache
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Set up Docker
+      uses: docker/setup-docker-action@v4
+    - name: Set up Docker Compose
+      uses: docker/setup-compose-action@v1
+    - name: Test (end-to-end)
+      env:
+        AIRTABLE_PERSONAL_ACCESS_TOKEN: ${{ secrets.AIRTABLE_PERSONAL_ACCESS_TOKEN }}
+        BOX_APP_CONFIG: ${{ secrets.BOX_APP_CONFIG }}
+        CONFLUENCE_API_TOKEN: ${{ secrets.CONFLUENCE_API_TOKEN }}
+        CONFLUENCE_USER_EMAIL: ${{ secrets.CONFLUENCE_USER_EMAIL }}
+        DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
+        DROPBOX_APP_KEY: ${{ secrets.DROPBOX_APP_KEY }}
+        DROPBOX_APP_SECRET: ${{ secrets.DROPBOX_APP_SECRET }}
+        DROPBOX_REFRESH_TOKEN: ${{ secrets.DROPBOX_REFRESH_TOKEN }}
+        GCP_INGEST_SERVICE_KEY: ${{ secrets.GCP_INGEST_SERVICE_KEY }}
+        GH_READ_ONLY_ACCESS_TOKEN: ${{ secrets.GH_READ_ONLY_ACCESS_TOKEN }}
+        HUBSPOT_API_TOKEN: ${{ secrets.HUBSPOT_API_TOKEN }}
+        JIRA_INGEST_API_TOKEN: ${{ secrets.JIRA_INGEST_API_TOKEN }}
+        JIRA_INGEST_USER_EMAIL: ${{ secrets.JIRA_INGEST_USER_EMAIL }}
+        MONGODB_URI: ${{ secrets.MONGODB_URI }}
+        MONGODB_DATABASE_NAME: ${{ secrets.MONGODB_DATABASE_NAME }}
+        MS_CLIENT_CRED: ${{ secrets.MS_CLIENT_CRED }}
+        MS_CLIENT_ID: ${{ secrets.MS_CLIENT_ID }}
+        MS_TENANT_ID: ${{ secrets.MS_TENANT_ID }}
+        MS_USER_EMAIL: ${{ secrets.MS_USER_EMAIL }}
+        MS_USER_PNAME: ${{ secrets.MS_USER_PNAME }}
+        SALESFORCE_USERNAME: ${{secrets.SALESFORCE_USERNAME}}
+        SALESFORCE_CONSUMER_KEY: ${{secrets.SALESFORCE_CONSUMER_KEY}}
+        SALESFORCE_PRIVATE_KEY: ${{secrets.SALESFORCE_PRIVATE_KEY}}
+        SHAREPOINT_CLIENT_ID: ${{secrets.SHAREPOINT_CLIENT_ID}}
+        SHAREPOINT_CRED: ${{secrets.SHAREPOINT_CRED}}
+        SHAREPOINT_SITE: ${{secrets.SHAREPOINT_SITE}}
+        SHAREPOINT_PERMISSIONS_APP_ID: ${{secrets.SHAREPOINT_PERMISSIONS_APP_ID}}
+        SHAREPOINT_PERMISSIONS_APP_CRED: ${{secrets.SHAREPOINT_PERMISSIONS_APP_CRED}}
+        SHAREPOINT_PERMISSIONS_TENANT: ${{secrets.SHAREPOINT_PERMISSIONS_TENANT}}
+        SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
+        UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
+        NOTION_API_KEY: ${{ secrets.NOTION_API_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }}
+        AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }}
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        OCTOAI_API_KEY: ${{ secrets.OCTOAI_API_KEY }}
+        PINECONE_API_KEY: ${{secrets.PINECONE_API_KEY}}
+        ASTRA_DB_APPLICATION_TOKEN: ${{secrets.ASTRA_DB_TOKEN}}
+        ASTRA_DB_API_ENDPOINT: ${{secrets.ASTRA_DB_ENDPOINT}}
+        MXBAI_API_KEY: ${{secrets.MXBAI_API_KEY}}
+        OCR_AGENT: "unstructured.partition.utils.ocr_models.tesseract_ocr.OCRAgentTesseract"
+        CI: "true"
+      run: |
+        retry() {
+          local n=1 max=5 delay=15
+          while true; do
+            "$@" && return 0
+            if (( n >= max )); then
+              echo "Command failed after $n attempts: $*" >&2
+              return 1
+            fi
+            echo "Attempt $n/$max failed for: $*. Retrying in ${delay}s..." >&2
+            sleep "$delay"
+            n=$((n+1))
+          done
+        }
+        retry sudo apt-get update
+        retry sudo apt-get install -y libmagic-dev poppler-utils libreoffice
+        retry sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
+        retry sudo apt-get update -o "APT::Update::Error-Mode=any"
+        retry sudo apt-get install -y tesseract-ocr
+        retry sudo apt-get install -y tesseract-ocr-kor
+        retry sudo apt-get install -y diffstat
+        tesseract --version
+        uv run --no-sync ./test_unstructured_ingest/test-ingest-src.sh
+
+  test_json_to_html:
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    runs-on: ubuntu-latest
+    needs: [setup, lint]
+    steps:
+    - uses: 'actions/checkout@v4'
+    - uses: ./.github/actions/base-cache
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Test HTML fixtures
+      env:
+        OVERWRITE_FIXTURES: "false"
+      run: |
+        sudo apt-get install diffstat
+        uv run --no-sync ./test_unstructured_ingest/check-diff-expected-output-html.sh
+
+  test_json_to_markdown:
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    runs-on: ubuntu-latest
+    needs: [setup, lint]
+    steps:
+    - uses: 'actions/checkout@v4'
+    - uses: ./.github/actions/base-cache
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Test markdown fixtures
+      env:
+        OVERWRITE_FIXTURES: "false"
+      run: |
+        sudo apt-get install diffstat
+        uv run --no-sync ./test_unstructured_ingest/check-diff-expected-output-markdown.sh
+
+  changelog:
+    runs-on: ubuntu-latest
+    steps:
+    # need to checkout otherwise paths-filter will fail on merge-queue trigger
+    - uses: actions/checkout@v4
+    - if: github.ref != 'refs/heads/main'
+      uses: dorny/paths-filter@v3
+      id: changes
+      with:
+        filters: |
+          src:
+            - 'unstructured/**'
+
+    - if: steps.changes.outputs.src == 'true' && github.ref != 'refs/heads/main'
+      uses: dangoslen/changelog-enforcer@v3
+
+  # TODO - figure out best practice for caching docker images
+  # (Using the virtualenv to get pytest)
+  test_dockerfile:
+    runs-on: opensource-linux-8core
+    needs: [ setup, lint ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Test Dockerfile
+        run: |
+          echo "UNS_API_KEY=${{ secrets.UNS_API_KEY }}" > uns_test_env_file
+          make docker-build
+          make docker-test CI=true UNSTRUCTURED_INCLUDE_DEBUG_METADATA=true
+      - name: Scan image
+        uses: anchore/scan-action@v3
+        with:
+          image: "unstructured:dev"
+          severity-cutoff: critical
+          fail-build: false
+          only-fixed: true
+          output-format: table

.github/workflows/claude.yml

@@ -0,0 +1,37 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
+      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@beta
+        with:
+          anthropic_api_key: ${{ secrets.GH_ANTHROPIC_API_KEY }}
+          allowed_tools: "Bash(git:*),View,GlobTool,GrepTool,BatchTool"

.github/workflows/codeflash.yml

@@ -0,0 +1,42 @@
+name: Codeflash Optimization
+
+on:
+  pull_request:
+    paths:
+      - 'unstructured/**' 
+
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  optimize:
+    name: Optimize new Python code
+    if: ${{ github.actor != 'codeflash-ai[bot]' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: 🐍 Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: 📦 Install Environment
+        uses: ./.github/actions/base-cache
+        with:
+          python-version: 3.12
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libmagic-dev poppler-utils libreoffice
+          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
+          sudo apt-get update
+          sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
+      - name: Codeflash Optimization
+        env:
+          UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
+          CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }}
+        run: uvx codeflash

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,76 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+    # known issue that codeql doesn't support merge_queue event, recommendation is to skip it on merge_queue
+    # https://github.com/github/codeql-action/issues/1572 and https://github.com/github/codeql-action/issues/1537
+  schedule:
+    - cron: '21 21 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/docker-publish.yml

@@ -0,0 +1,107 @@
+name: Build And Push Docker Image
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  DOCKER_REPOSITORY: quay.io/unstructured-io/unstructured
+  DOCKER_BUILD_REPOSITORY: quay.io/unstructured-io/build-unstructured
+
+jobs:
+  set-short-sha:
+    runs-on: ubuntu-latest
+    outputs:
+      short_sha: ${{ steps.set_short_sha.outputs.short_sha }}
+    steps:
+      - name: Set Short SHA
+        id: set_short_sha
+        run: echo "short_sha=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
+
+  build-images:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - docker-platform: "linux/amd64"
+            runs-on: opensource-linux-8core
+          - docker-platform: "linux/arm64"
+            runs-on: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runs-on }}
+    needs: set-short-sha
+    env:
+      SHORT_SHA: ${{ needs.set-short-sha.outputs.short_sha }}
+    steps:
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v3
+      with:
+        driver: docker
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Login to Quay.io
+      uses: docker/login-action@v3
+      with:
+        registry: quay.io
+        username: ${{ secrets.QUAY_IO_ROBOT_USERNAME }}
+        password: ${{ secrets.QUAY_IO_ROBOT_TOKEN }}
+    - name: Build images
+      run: |
+        ARCH=$(cut -d "/" -f2 <<< ${{ matrix.docker-platform }})
+        DOCKER_BUILDKIT=1 docker buildx build --platform=${{ matrix.docker-platform }} --load \
+          -f Dockerfile \
+          --build-arg BUILDKIT_INLINE_CACHE=1 \
+          --progress plain \
+          --cache-from $DOCKER_BUILD_REPOSITORY:$ARCH \
+          -t $DOCKER_BUILD_REPOSITORY:$ARCH-$SHORT_SHA .
+    - name: Test images
+      run: |
+        echo "UNS_API_KEY=${{ secrets.UNS_API_KEY }}" > uns_test_env_file
+        echo "UNSTRUCTURED_HF_TOKEN=${{ secrets.HF_TOKEN }}" >> uns_test_env_file
+        ARCH=$(cut -d "/" -f2 <<< ${{ matrix.docker-platform }})
+        DOCKER_IMAGE="$DOCKER_BUILD_REPOSITORY:$ARCH-$SHORT_SHA" \
+          make docker-test CI=true TEST_FILE=test_unstructured/partition/test_text.py
+        DOCKER_IMAGE=$DOCKER_BUILD_REPOSITORY:$ARCH-$SHORT_SHA make docker-smoke-test
+    - name: Push images
+      run: |
+        # write to the build repository to cache for the publish-images job
+        ARCH=$(cut -d "/" -f2 <<< ${{ matrix.docker-platform }})
+        docker push "$DOCKER_BUILD_REPOSITORY:$ARCH-$SHORT_SHA"
+  publish-images:
+    runs-on: ubuntu-latest-m
+    needs: [set-short-sha, build-images]
+    env:
+      SHORT_SHA: ${{ needs.set-short-sha.outputs.short_sha }}
+    steps:
+    - uses: docker/setup-buildx-action@v3
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Login to Quay.io
+      uses: docker/login-action@v3
+      with:
+        registry: quay.io
+        username: ${{ secrets.QUAY_IO_ROBOT_USERNAME }}
+        password: ${{ secrets.QUAY_IO_ROBOT_TOKEN }}
+    - name: Pull AMD image
+      run: |
+        docker pull $DOCKER_BUILD_REPOSITORY:amd64-$SHORT_SHA
+    - name: Pull ARM image
+      run: |
+        docker pull $DOCKER_BUILD_REPOSITORY:arm64-$SHORT_SHA
+    - name: Push latest build tags for AMD and ARM
+      run: |
+        # these are used to construct the final manifest but also cache-from in subsequent runs
+        docker tag $DOCKER_BUILD_REPOSITORY:amd64-$SHORT_SHA $DOCKER_BUILD_REPOSITORY:amd64
+        docker push $DOCKER_BUILD_REPOSITORY:amd64
+        docker tag $DOCKER_BUILD_REPOSITORY:arm64-$SHORT_SHA $DOCKER_BUILD_REPOSITORY:arm64
+        docker push $DOCKER_BUILD_REPOSITORY:arm64
+    - name: Push multiarch manifest
+      run: |
+        docker manifest create ${DOCKER_REPOSITORY}:latest $DOCKER_BUILD_REPOSITORY:amd64 $DOCKER_BUILD_REPOSITORY:arm64
+        docker manifest push $DOCKER_REPOSITORY:latest
+        docker manifest create ${DOCKER_REPOSITORY}:$SHORT_SHA $DOCKER_BUILD_REPOSITORY:amd64 $DOCKER_BUILD_REPOSITORY:arm64
+        docker manifest push $DOCKER_REPOSITORY:$SHORT_SHA
+        VERSION=$(grep -Po '(?<=__version__ = ")[^"]*' unstructured/__version__.py)
+        docker manifest create ${DOCKER_REPOSITORY}:$VERSION $DOCKER_BUILD_REPOSITORY:amd64 $DOCKER_BUILD_REPOSITORY:arm64
+        docker manifest push $DOCKER_REPOSITORY:$VERSION

.github/workflows/ingest-test-fixtures-update-pr.yml

@@ -0,0 +1,130 @@
+name: Ingest Test Fixtures Update PR
+
+on:
+  workflow_dispatch:
+
+env:
+  PYTHON_VERSION: "3.12"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/base-cache
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+  update-fixtures-and-pr:
+    runs-on: ubuntu-latest
+    needs: [setup]
+    steps:
+      # actions/checkout MUST come before auth
+      - uses: "actions/checkout@v4"
+      - uses: ./.github/actions/base-cache
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Set up Docker Compose
+        uses: docker/setup-compose-action@v1
+      - name: Update test fixtures
+        env:
+          AIRTABLE_PERSONAL_ACCESS_TOKEN: ${{ secrets.AIRTABLE_PERSONAL_ACCESS_TOKEN }}
+          BOX_APP_CONFIG: ${{ secrets.BOX_APP_CONFIG }}
+          CONFLUENCE_API_TOKEN: ${{ secrets.CONFLUENCE_API_TOKEN }}
+          CONFLUENCE_USER_EMAIL: ${{ secrets.CONFLUENCE_USER_EMAIL }}
+          DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
+          DROPBOX_APP_KEY: ${{ secrets.DROPBOX_APP_KEY }}
+          DROPBOX_APP_SECRET: ${{ secrets.DROPBOX_APP_SECRET }}
+          DROPBOX_REFRESH_TOKEN: ${{ secrets.DROPBOX_REFRESH_TOKEN }}
+          GCP_INGEST_SERVICE_KEY: ${{ secrets.GCP_INGEST_SERVICE_KEY }}
+          GH_READ_ONLY_ACCESS_TOKEN: ${{ secrets.GH_READ_ONLY_ACCESS_TOKEN }}
+          HUBSPOT_API_TOKEN: ${{ secrets.HUBSPOT_API_TOKEN }}
+          JIRA_INGEST_API_TOKEN: ${{ secrets.JIRA_INGEST_API_TOKEN }}
+          JIRA_INGEST_USER_EMAIL: ${{ secrets.JIRA_INGEST_USER_EMAIL }}
+          MONGODB_URI: ${{ secrets.MONGODB_URI }}
+          MONGODB_DATABASE_NAME: ${{ secrets.MONGODB_DATABASE_NAME }}
+          MS_CLIENT_CRED: ${{ secrets.MS_CLIENT_CRED }}
+          MS_CLIENT_ID: ${{ secrets.MS_CLIENT_ID }}
+          MS_TENANT_ID: ${{ secrets.MS_TENANT_ID }}
+          MS_USER_EMAIL: ${{ secrets.MS_USER_EMAIL }}
+          MS_USER_PNAME: ${{ secrets.MS_USER_PNAME }}
+          SALESFORCE_USERNAME: ${{secrets.SALESFORCE_USERNAME}}
+          SALESFORCE_CONSUMER_KEY: ${{secrets.SALESFORCE_CONSUMER_KEY}}
+          SALESFORCE_PRIVATE_KEY: ${{secrets.SALESFORCE_PRIVATE_KEY}}
+          SHAREPOINT_CLIENT_ID: ${{secrets.SHAREPOINT_CLIENT_ID}}
+          SHAREPOINT_CRED: ${{secrets.SHAREPOINT_CRED}}
+          SHAREPOINT_SITE: ${{secrets.SHAREPOINT_SITE}}
+          SHAREPOINT_PERMISSIONS_APP_ID: ${{secrets.SHAREPOINT_PERMISSIONS_APP_ID}}
+          SHAREPOINT_PERMISSIONS_APP_CRED: ${{secrets.SHAREPOINT_PERMISSIONS_APP_CRED}}
+          SHAREPOINT_PERMISSIONS_TENANT: ${{secrets.SHAREPOINT_PERMISSIONS_TENANT}}
+          SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
+          UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
+          NOTION_API_KEY: ${{ secrets.NOTION_API_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }}
+          AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          OCTOAI_API_KEY: ${{ secrets.OCTOAI_API_KEY }}
+          ASTRA_DB_APPLICATION_TOKEN: ${{secrets.ASTRA_DB_TOKEN}}
+          ASTRA_DB_API_ENDPOINT: ${{secrets.ASTRA_DB_ENDPOINT}}
+          PINECONE_API_KEY: ${{secrets.PINECONE_API_KEY}}
+          MXBAI_API_KEY: ${{secrets.MXBAI_API_KEY}}
+          OCR_AGENT: "unstructured.partition.utils.ocr_models.tesseract_ocr.OCRAgentTesseract"
+          OVERWRITE_FIXTURES: "true"
+          CI: "true"
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libmagic-dev poppler-utils libreoffice
+          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
+          sudo apt-get update
+          sudo apt-get install -y tesseract-ocr
+          sudo apt-get install -y tesseract-ocr-kor
+          sudo apt-get install diffstat
+          tesseract --version
+          uv run --no-sync ./test_unstructured_ingest/test-ingest-src.sh
+      - name: Update HTML fixtures
+        run: make html-fixtures-update
+      - name: Update markdown fixtures
+        run: make markdown-fixtures-update
+
+      - name: Save branch name to environment file
+        id: branch
+        run: |
+          original_branch=$(git rev-parse --abbrev-ref HEAD)
+          suffix="|ingest-test-fixtures-update-$(git rev-parse --short HEAD)"
+          branch_name="$original_branch$suffix"
+          echo "BRANCH_NAME=$branch_name" >> $GITHUB_ENV
+
+      - name: Save PR name to environment file
+        id: pr
+        run: |
+          commit_sha=$(git rev-parse HEAD)
+          prs=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/${{ github.repository }}/commits/${commit_sha}/pulls")
+          pr_name=$(echo "$prs" | jq -r '.[0].title')
+          echo "PR_NAME=$pr_name" >> $GITHUB_ENV
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.GH_CREATE_PR_TOKEN }}
+          add-paths: |
+            test_unstructured_ingest/expected-structured-output
+            test_unstructured_ingest/expected-structured-output-html
+            test_unstructured_ingest/expected-structured-output-markdown
+            test_unstructured_ingest/metrics
+          commit-message: "Update ingest test fixtures"
+          branch: ${{ env.BRANCH_NAME }}
+          title: "${{ env.PR_NAME }} <- Ingest test fixtures update"
+          assignees: ${{ github.actor }}
+          reviewers: ${{ github.actor }}
+          delete-branch: true
+          body: |
+            This pull request includes updated ingest test fixtures.
+            Please review and merge if appropriate.
+          base: ${{ github.head_ref }}

.github/workflows/partition-benchmark.yaml

@@ -0,0 +1,172 @@
+name: Partition Benchmark
+
+# PRs are gated against a rolling median baseline; pushes to main record a new
+# baseline sample. A manual run (workflow_dispatch) on main is a read-only
+# inspection; a manual run on any OTHER branch is a self-test that records +
+# uploads to a throwaway `-selftest` prefix -- so the write path can be validated
+# on a branch, before merge, without touching the real baseline. (Keying on the
+# ref rather than an input is deliberate: dispatch inputs are validated against
+# the default-branch workflow, so a branch-only input can't be passed pre-merge.)
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  NLTK_DATA: ${{ github.workspace }}/nltk_data
+  PYTHON_VERSION: "3.12"
+  # Number of times to run the full benchmark suite.
+  NUM_ITERATIONS: "3"
+  # Allowance over the rolling median baseline. Sized to absorb runner-speed
+  # variance on shared ubuntu-latest runners (observed ~1.6x), not just a code delta.
+  REGRESSION_THRESHOLD: "0.30"
+  # How many recent main-run samples the baseline median is taken over.
+  BASELINE_WINDOW: "20"
+  # Below this many samples, the gate is in warm-up (records, does not fail).
+  BASELINE_MIN_SAMPLES: "5"
+  # Increment to change cache key when benchmark-affecting dependencies are updated, to ensure clean slate runs.
+  CACHE_VERSION: "v2"
+  # S3 location for metrics – matches core-product convention.
+  S3_METRICS_BUCKET_KEY: utic-metrics/ci-metrics
+  # Prefix holding one immutable JSON record per main run (the rolling history).
+  S3_HISTORY_PREFIX: open-source/partition-benchmark/history
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/base-cache
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+  benchmark:
+    name: Measure and compare partition() runtime
+    runs-on: ubuntu-latest
+    needs: [setup]
+
+    steps:
+
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/base-cache
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libmagic-dev poppler-utils libreoffice
+          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
+          sudo apt-get update
+          sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
+
+
+      - name: Restore HuggingFace model cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/.cache/huggingface
+          key: hf-models-${{ runner.os }}-${{ env.CACHE_VERSION }}-${{ github.sha }}
+          restore-keys: |
+            hf-models-${{ runner.os }}-${{ env.CACHE_VERSION }}-
+            hf-models-${{ runner.os }}-
+
+
+      - name: Run partition benchmark
+        env:
+          NUM_ITERATIONS: ${{ env.NUM_ITERATIONS }}
+        run: |
+          uv run --no-sync python scripts/performance/benchmark_partition.py \
+            benchmark_results.json
+
+      - name: Save HuggingFace model cache
+        uses: actions/cache/save@v4
+        with:
+          path: ~/.cache/huggingface
+          key: hf-models-${{ runner.os }}-${{ env.CACHE_VERSION }}-${{ github.sha }}
+
+
+      # Decide, in one place, which baseline mode this run is in:
+      #   push:main                          -> record + upload to the real prefix
+      #   workflow_dispatch on a non-main ref -> record + upload to a throwaway -selftest prefix
+      #   pull_request / dispatch on main     -> read-only, real prefix (gate only, never write)
+      - name: Resolve baseline mode
+        id: mode
+        run: |
+          prefix="${{ env.S3_HISTORY_PREFIX }}"
+          record=""
+          upload="false"
+          if [ "${{ github.event_name }}" = "push" ]; then
+            record="--record"; upload="true"
+          elif [ "${{ github.event_name }}" = "workflow_dispatch" ] && \
+               [ "${{ github.ref_name }}" != "main" ]; then
+            prefix="${prefix}-selftest"; record="--record"; upload="true"
+          fi
+          {
+            echo "prefix=$prefix"
+            echo "record=$record"
+            echo "upload=$upload"
+          } >> "$GITHUB_OUTPUT"
+          echo "mode: prefix=$prefix record='$record' upload=$upload"
+
+
+      # Pull the whole history prefix (one object per past run). On fork PRs the S3
+      # secrets are absent, so this no-ops -> empty dir -> warm-up/pass.
+      - name: Download benchmark history from S3
+        continue-on-error: true
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_EVAL_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_EVAL_SECRET_KEY }}
+        run: |
+          mkdir -p bench_history
+          aws s3 cp --recursive \
+            "s3://${{ env.S3_METRICS_BUCKET_KEY }}/${{ steps.mode.outputs.prefix }}/" \
+            bench_history/
+
+
+      # Gate against the rolling median. When recording (push, or dispatch self-test),
+      # --record also writes this run's record into bench_history/ for the upload below.
+      - name: Compare against rolling baseline
+        id: compare
+        run: |
+          uv run --no-sync python scripts/performance/compare_benchmark.py \
+            benchmark_results.json \
+            bench_history \
+            --threshold ${{ env.REGRESSION_THRESHOLD }} \
+            --window ${{ env.BASELINE_WINDOW }} \
+            --min-samples ${{ env.BASELINE_MIN_SAMPLES }} \
+            ${{ steps.mode.outputs.record }}
+
+
+      # Extend the baseline. `sync` (no --delete) uploads the one new record written
+      # above; existing objects are unchanged. PR/read-only runs skip this entirely.
+      # `!cancelled()` (instead of the implicit success() &&) so the record still
+      # persists when the compare step fails the gate on a regression -- otherwise a
+      # red run never reaches S3 and the baseline can't track the new reality. The
+      # record is written before the gate check, so the local file exists either way.
+      - name: Upload new baseline record to S3
+        if: ${{ !cancelled() && steps.mode.outputs.upload == 'true' }}
+        continue-on-error: true
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_EVAL_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_EVAL_SECRET_KEY }}
+        run: |
+          aws s3 sync \
+            bench_history/ \
+            "s3://${{ env.S3_METRICS_BUCKET_KEY }}/${{ steps.mode.outputs.prefix }}/"
+
+
+      - name: Upload benchmark artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results-${{ github.sha }}
+          path: |
+            benchmark_results.json
+            bench_history/
+          retention-days: 30

.github/workflows/release-version-alert.yml

@@ -0,0 +1,83 @@
+name: Release Version Alert
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches: [ main ]
+
+jobs:
+  check-version:  
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout codes
+        uses: actions/checkout@v4 
+      - name: Get PR information
+        id: pr-info
+        run: |
+          PR_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")
+          HAS_PR=false; [ "$PR_NUMBER" != "null" ] && HAS_PR=true
+          echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV
+          echo "HAS_PR=$HAS_PR" >> $GITHUB_ENV
+          echo "PR_NUMBER=$PR_NUMBER"
+          echo "HAS_PR=$HAS_PR"
+      - name: Check versions
+        id: check-versions
+        run: |
+          CHECK_NEW_VERSION_RESPONSE=$(bash scripts/check-new-release-version.sh)
+          if [[ "$CHECK_NEW_VERSION_RESPONSE" == "New release version"* ]]; then
+            if [ "$HAS_PR" = true ]; then
+              MESSAGE="$CHECK_NEW_VERSION_RESPONSE :rocket: Coming soon in PR: https://github.com/$GITHUB_REPOSITORY/pull/$PR_NUMBER "
+            else
+              BRANCH_NAME=$(echo "${GITHUB_REF#refs/heads/}")
+              BRANCH_LINK="https://github.com/${{ github.repository }}/tree/$BRANCH_NAME"
+              MESSAGE="$CHECK_NEW_VERSION_RESPONSE :rocket: Coming soon in branch: $BRANCH_LINK"
+            fi
+            echo "SLACK_MESSAGE=$MESSAGE" >> $GITHUB_ENV
+          else
+            echo "No new non-dev version found. Skipping Slack notification."
+            echo "SKIP_STEPS=true" >> $GITHUB_ENV  # Set an environment variable to indicate skipping steps
+          fi 
+          echo "SLACK_MESSAGE=$MESSAGE"
+      - name: Generate Message Hash
+        if: env.SKIP_STEPS != 'true' 
+        id: generate-hash
+        run: |
+          MESSAGE_HASH=$(echo "${{env.SLACK_MESSAGE}}" | sha256sum | cut -d ' ' -f1)
+          echo "MESSAGE_HASH=$MESSAGE_HASH" >> $GITHUB_ENV
+      - name: Restore Message from Cache
+        if: env.SKIP_STEPS != 'true'
+        id: restore-cache
+        uses: actions/cache/restore@v4
+        with: 
+          path: message_cache.txt
+          key: message-cache-${{ env.MESSAGE_HASH  }}
+      - name: Check for Duplicates
+        if: env.SKIP_STEPS != 'true'
+        run: |
+          DUPLICATE_CHECK=$(grep -Fx "${{env.SLACK_MESSAGE}}" message_cache.txt || true)
+          echo "DUPLICATE_CHECK=$DUPLICATE_CHECK"
+          if [ -n "$DUPLICATE_CHECK" ]; then
+            echo "Message already posted. Skipping duplicate Slack notification."
+            echo "SKIP_STEPS=true" >> $GITHUB_ENV  # Set an environment variable to indicate skipping steps
+          fi
+      - name: Write Message to Cache File
+        if: env.SKIP_STEPS != 'true'
+        run: |
+          echo "${{env.SLACK_MESSAGE}}" >> message_cache.txt
+          cat message_cache.txt
+      - name: Store Message in Cache
+        if: env.SKIP_STEPS != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: message_cache.txt
+          key: message-cache-${{ env.MESSAGE_HASH }}
+      - name: Slack Notification
+        if: env.SKIP_STEPS != 'true'
+        uses: slackapi/slack-github-action@v1.24.0
+        with:
+          channel-id: 'C05S1QMKL5D'
+          slack-message: ${{ env.SLACK_MESSAGE }}
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,93 @@
+name: Pypi Release
+
+on:
+  release:
+    types:
+      - published
+
+permissions:
+  contents: read
+  id-token: write       # Required for PyPI trusted publishing / attestations
+
+concurrency:
+  group: release
+  cancel-in-progress: false
+
+env:
+  PYTHON_VERSION: "3.12"
+
+jobs:
+  release:
+    # Skip opencv wheel releases (tagged 'opencv-*') produced by the
+    # build-opencv-wheels workflow - those aren't package releases and would
+    # otherwise trigger a spurious PyPI publish failure.
+    if: ${{ !startsWith(github.event.release.tag_name, 'opencv-') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Set up Python
+        run: uv python install
+
+      - name: Install dependencies
+        run: uv sync --locked --only-group release --no-install-project
+
+      - name: Validate version matches release tag
+        env:
+          TAG: ${{ github.event.release.tag_name }}
+        run: |
+          PKG_VERSION=$(uv run --script - <<'PYEOF'
+          # /// script
+          # requires-python = ">=3.11"
+          # dependencies = ["typing-extensions", "requests"]
+          # ///
+          from unstructured.__version__ import __version__
+          print(__version__)
+          PYEOF
+          )
+          if [[ "$TAG" != "$PKG_VERSION" && "$TAG" != "v$PKG_VERSION" ]]; then
+            echo "Tag '$TAG' does not match package version '$PKG_VERSION'"
+            exit 1
+          fi
+
+      - name: Build artifact
+        id: build
+        run: uv build
+
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
+
+      # Best-effort: attempt Azure upload even if PyPI fails, but only if build succeeded.
+      - name: Create .pypirc for Azure Artifacts
+        if: always() && steps.build.outcome == 'success'
+        run: |
+          cat <<EOF > ~/.pypirc
+          [distutils]
+          index-servers =
+              azure
+
+          [azure]
+          repository: https://pkgs.dev.azure.com/${{ secrets.AZURE_ARTIFACTS_FEED }}/_packaging/${{ secrets.AZURE_ARTIFACTS_FEED }}/pypi/upload/
+          username: ${{ secrets.AZURE_ARTIFACTS_USERNAME }}
+          password: ${{ secrets.AZURE_ARTIFACTS_PAT }}
+          EOF
+
+      - name: Publish package to Azure Artifacts
+        if: always() && steps.build.outcome == 'success'
+        run: |
+          EXIT_CODE=0
+          uv run --no-sync twine upload -r azure dist/* || EXIT_CODE=$?
+          if [[ $EXIT_CODE -eq 0 ]]; then
+            echo "Successfully published to Azure Artifacts (or already existed)"
+          else
+            echo "Azure Artifacts upload failed (exit code: $EXIT_CODE)"
+            if [[ $EXIT_CODE -eq 1 ]]; then
+              echo "This may be due to version conflicts or connectivity issues"
+            fi
+            echo "Azure Artifacts upload is non-critical - skipping failure"
+          fi