chore: import upstream snapshot with attribution

.github/workflows/docker.yml

@@ -0,0 +1,86 @@
+name: docker
+
+on:
+  push:
+    tags: ["v*.*.*"]
+  workflow_dispatch:
+    inputs:
+      ludwig_version:
+        description: "PyPI version to install, e.g. 0.13.0 (leave empty to build from source)"
+        required: false
+        default: ""
+      latest:
+        description: "Also tag as :latest"
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  docker:
+    name: Build docker image ${{ matrix.docker-image }}
+    if: github.repository == 'ludwig-ai/ludwig'
+    runs-on: ubuntu-latest
+
+    # cancel in-progress runs for the same branch/tag
+    concurrency:
+      group: docker-${{ matrix.docker-image }}-${{ inputs.ludwig_version || github.head_ref || github.sha }}
+      cancel-in-progress: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        docker-image:
+          - ludwig
+          - ludwig-gpu
+          - ludwig-ray
+          - ludwig-ray-gpu
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Derive minor version tag
+        id: version
+        if: inputs.ludwig_version != ''
+        run: |
+          minor=$(echo "${{ inputs.ludwig_version }}" | cut -d. -f1-2)
+          echo "minor=${minor}" >> "$GITHUB_OUTPUT"
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v6
+        with:
+          images: |
+            ludwigai/${{ matrix.docker-image }}
+          tags: |
+            # versioned dispatch: use explicit PyPI version tags
+            type=raw,value=${{ inputs.ludwig_version }},enable=${{ inputs.ludwig_version != '' }}
+            type=raw,value=${{ steps.version.outputs.minor }},enable=${{ inputs.ludwig_version != '' }}
+            type=raw,value=latest,enable=${{ inputs.ludwig_version != '' && inputs.latest }}
+            # tag push: derive tags from git ref
+            type=semver,pattern={{version}},enable=${{ inputs.ludwig_version == '' }}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ inputs.ludwig_version == '' }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Login to DockerHub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ./docker/${{ matrix.docker-image }}/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            LUDWIG_VERSION=${{ inputs.ludwig_version }}

.github/workflows/pytest.yml

@@ -0,0 +1,267 @@
+name: pytest
+
+on:
+  push:
+    branches: ["main", "release-*", "future-capabilities"]
+  pull_request:
+    branches: ["main", "release-*", "future-capabilities"]
+
+concurrency:
+  group: pytest-${{ github.head_ref || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.LUDWIG_TESTS_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.LUDWIG_TESTS_AWS_SECRET_ACCESS_KEY }}
+      KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
+      KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
+      IS_NOT_FORK: ${{ !(github.event.pull_request.base.repo.full_name == 'ludwig-ai/ludwig' && github.event.pull_request.head.repo.fork) }}
+      UV_SYSTEM_PYTHON: "1"
+
+    name: Unit Tests
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup Linux
+        run: |
+          sudo apt-get update && sudo apt-get install -y cmake libsndfile1 libsox-dev ffmpeg
+
+      - name: uv cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-uv-unit-${{ hashFiles('pyproject.toml', '.github/workflows/pytest.yml') }}
+
+      - name: Install uv
+        run: pip install uv
+
+      - name: Install dependencies
+        run: |
+          uv pip install torch==2.12.0 torchvision==0.27.0 torchaudio==2.11.0 torchcodec --extra-index-url https://download.pytorch.org/whl/cpu
+          uv pip install '.[test]'
+          uv pip list
+
+      - name: Unit Tests
+        run: |
+          RUN_PRIVATE=$IS_NOT_FORK LUDWIG_TEST_SUITE_TIMEOUT_S=5400 pytest -v --timeout 300 --durations 100 -m "not distributed and not slow and not combinatorial and not llm" --junitxml pytest.xml tests/ludwig
+
+      - name: Regression Tests
+        run: |
+          RUN_PRIVATE=$IS_NOT_FORK LUDWIG_TEST_SUITE_TIMEOUT_S=5400 pytest -v --timeout 300 --durations 100 -m "not distributed and not slow and not combinatorial and not llm" --junitxml pytest-regression.xml tests/regression_tests
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Unit Test Results
+          path: pytest*.xml
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test-markers:
+          - "integration_tests_a"
+          - "integration_tests_b"
+          - "integration_tests_c"
+          - "integration_tests_d"
+          - "integration_tests_e"
+          - "integration_tests_f"
+          - "integration_tests_g"
+          - "integration_tests_h"
+          - "integration_tests_i"
+          - "integration_tests_j"
+
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.LUDWIG_TESTS_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.LUDWIG_TESTS_AWS_SECRET_ACCESS_KEY }}
+      KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
+      KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
+      IS_NOT_FORK: ${{ !(github.event.pull_request.base.repo.full_name == 'ludwig-ai/ludwig' && github.event.pull_request.head.repo.fork) }}
+      MARKERS: ${{ matrix.test-markers }}
+      UV_SYSTEM_PYTHON: "1"
+      MLFLOW_ALLOW_FILE_STORE: "true"
+
+    name: Integration (${{ matrix.test-markers }})
+    services:
+      minio:
+        image: fclairamb/minio-github-actions
+        env:
+          MINIO_ACCESS_KEY: minio
+          MINIO_SECRET_KEY: minio123
+        ports:
+          - 9000:9000
+
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup Linux
+        run: |
+          sudo apt-get update && sudo apt-get install -y cmake libsndfile1 libsox-dev ffmpeg
+
+      - name: uv cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-uv-integration-${{ hashFiles('pyproject.toml', '.github/workflows/pytest.yml') }}
+
+      - name: Install dependencies
+        run: |
+          pip install uv
+          uv pip install torch==2.12.0 torchvision==0.27.0 torchaudio==2.11.0 torchcodec --extra-index-url https://download.pytorch.org/whl/cpu
+          uv pip install '.[test]'
+          uv pip list
+
+      - name: Free Disk Space
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: true
+          swap-storage: true
+
+      - name: Integration Tests
+        run: |
+          RUN_PRIVATE=$IS_NOT_FORK LUDWIG_TEST_SUITE_TIMEOUT_S=7200 pytest -v --timeout 300 --durations 100 -m "not slow and not combinatorial and not llm and $MARKERS" --junitxml pytest.xml tests/integration_tests
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Integration Test Results (${{ matrix.test-markers }})
+          path: pytest.xml
+
+  distributed-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        distributed-group:
+          - "distributed_a"
+          - "distributed_b"
+          - "distributed_c"
+          - "distributed_d"
+          - "distributed_e"
+          - "distributed_f"
+
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.LUDWIG_TESTS_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.LUDWIG_TESTS_AWS_SECRET_ACCESS_KEY }}
+      KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
+      KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
+      IS_NOT_FORK: ${{ !(github.event.pull_request.base.repo.full_name == 'ludwig-ai/ludwig' && github.event.pull_request.head.repo.fork) }}
+      DIST_GROUP: ${{ matrix.distributed-group }}
+      UV_SYSTEM_PYTHON: "1"
+      MLFLOW_ALLOW_FILE_STORE: "true"
+
+    name: Distributed (${{ matrix.distributed-group }})
+    services:
+      minio:
+        image: fclairamb/minio-github-actions
+        env:
+          MINIO_ACCESS_KEY: minio
+          MINIO_SECRET_KEY: minio123
+        ports:
+          - 9000:9000
+
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup Linux
+        run: |
+          sudo apt-get update && sudo apt-get install -y cmake libsndfile1 libsox-dev ffmpeg
+
+      - name: uv cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-uv-distributed-${{ hashFiles('pyproject.toml', '.github/workflows/pytest.yml') }}
+
+      - name: Install dependencies
+        run: |
+          pip install uv
+          uv pip install torch==2.12.0 torchvision==0.27.0 torchaudio==2.11.0 torchcodec --extra-index-url https://download.pytorch.org/whl/cpu
+          uv pip install '.[test]'
+          uv pip list
+
+      - name: Free Disk Space
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: true
+          swap-storage: true
+
+      - name: Distributed Tests
+        run: |
+          RUN_PRIVATE=$IS_NOT_FORK LUDWIG_TEST_SUITE_TIMEOUT_S=5400 pytest -v --timeout 300 --durations 100 -m "distributed and not slow and not combinatorial and not llm and $DIST_GROUP" --ignore=tests/integration_tests/test_server.py --junitxml pytest.xml tests/ludwig tests/integration_tests
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Distributed Test Results (${{ matrix.distributed-group }})
+          path: pytest.xml
+
+  test-minimal-install:
+    name: Minimal Install
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      UV_SYSTEM_PYTHON: "1"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup Linux
+        run: |
+          sudo apt-get update && sudo apt-get install -y cmake libsndfile1
+
+      - name: Install dependencies
+        run: |
+          pip install uv
+          uv pip install torch==2.12.0 torchvision==0.27.0 torchaudio==2.11.0 torchcodec --extra-index-url https://download.pytorch.org/whl/cpu
+          uv pip install .
+          uv pip list
+
+      - name: Check Install
+        run: |
+          ludwig check_install
+
+  event_file:
+    name: "Event File"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: Event File
+          path: ${{ github.event_path }}

.github/workflows/pytest_slow.yml

@@ -0,0 +1,72 @@
+# This workflow will install Python dependencies and run all tests marked as `slow` on a single Python version.
+
+name: pytest (slow)
+
+on:
+  push:
+    branches: ["main", "release-*"]
+
+jobs:
+  slow-pytest:
+    name: py-slow
+    runs-on: ubuntu-latest
+    env:
+      # Use Minio credentials for all S3 operations in tests.
+      # PyArrow/Ray S3 clients use these env vars directly, so they must point to Minio.
+      AWS_ACCESS_KEY_ID: minio
+      AWS_SECRET_ACCESS_KEY: minio123
+      AWS_ENDPOINT_URL: http://localhost:9000
+      KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
+      KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
+      IS_NOT_FORK: ${{ !(github.event.pull_request.base.repo.full_name == 'ludwig-ai/ludwig' && github.event.pull_request.head.repo.fork) }}
+      MLFLOW_ALLOW_FILE_STORE: "true"
+
+    services:
+      minio:
+        image: fclairamb/minio-github-actions
+        env:
+          MINIO_ACCESS_KEY: minio
+          MINIO_SECRET_KEY: minio123
+        ports:
+          - 9000:9000
+
+    timeout-minutes: 150
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup Linux
+        if: runner.os == 'linux'
+        run: |
+          sudo apt-get update && sudo apt-get install -y cmake libsndfile1 libsox-dev ffmpeg
+
+      - name: Install dependencies
+        run: |
+          python --version
+          pip --version
+          pip install uv
+          uv pip install --system torch==2.12.0 torchvision==0.27.0 torchaudio==2.11.0 torchcodec --extra-index-url https://download.pytorch.org/whl/cpu
+          uv pip install --system '.[test]'
+          pip list
+        shell: bash
+
+      - name: Create Minio test bucket
+        run: |
+          python -c "
+          import boto3
+          s3 = boto3.client('s3', endpoint_url='http://localhost:9000',
+                            aws_access_key_id='minio', aws_secret_access_key='minio123')
+          try:
+              s3.create_bucket(Bucket='ludwig-tests')
+          except s3.exceptions.BucketAlreadyOwnedByYou:
+              pass
+          "
+        shell: bash
+
+      - name: Tests
+        run: |
+          RUN_PRIVATE=$IS_NOT_FORK LUDWIG_TEST_SUITE_TIMEOUT_S=7200 pytest -v --timeout 600 --durations 100 -m "slow" --junitxml pytest.xml tests/integration_tests/

.github/workflows/schema.yml

@@ -0,0 +1,73 @@
+name: Publish JSON Schema
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "ludwig/schema/**"
+      - "ludwig/config_validation/**"
+      - "ludwig/constants.py"
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  publish-schema:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Ludwig
+        run: pip install -e ".[test]"
+
+      - name: Export schemas
+        run: |
+          mkdir -p schema-out
+          ludwig export_schema --model-type combined -o schema-out/ludwig-config.json
+          ludwig export_schema --model-type ecd -o schema-out/ludwig-config-ecd.json
+          ludwig export_schema --model-type llm -o schema-out/ludwig-config-llm.json
+
+      - name: Generate index.html
+        run: |
+          cat > schema-out/index.html << 'HTMLEOF'
+          <!DOCTYPE html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8">
+            <title>Ludwig JSON Schema</title>
+            <style>body{font-family:system-ui,sans-serif;max-width:640px;margin:2rem auto;padding:0 1rem}a{color:#0066cc}</style>
+          </head>
+          <body>
+            <h1>Ludwig JSON Schema</h1>
+            <p>JSON Schema files for <a href="https://github.com/ludwig-ai/ludwig">Ludwig</a> config validation and IDE auto-complete.</p>
+            <ul>
+              <li><a href="ludwig-config.json">ludwig-config.json</a> — Combined (ECD + LLM)</li>
+              <li><a href="ludwig-config-ecd.json">ludwig-config-ecd.json</a> — ECD only</li>
+              <li><a href="ludwig-config-llm.json">ludwig-config-llm.json</a> — LLM only</li>
+            </ul>
+            <h2>Usage</h2>
+            <p>Add to your Ludwig YAML config:</p>
+            <pre># yaml-language-server: $schema=https://ludwig-ai.github.io/schema/ludwig-config.json</pre>
+            <p>Or see <a href="https://www.schemastore.org/json/">SchemaStore</a> for automatic IDE integration.</p>
+          </body>
+          </html>
+          HTMLEOF
+
+      - name: Publish to ludwig-ai/schema
+        uses: cpina/github-action-push-to-another-repository@v1.7.2
+        env:
+          SSH_DEPLOY_KEY: ${{ secrets.SCHEMA_REPO_DEPLOY_KEY }}
+        with:
+          source-directory: schema-out
+          destination-github-username: ludwig-ai
+          destination-repository-name: schema
+          target-branch: main
+          commit-message: "Update Ludwig JSON schema from ${{ github.sha }}"

.github/workflows/test-results.yml

@@ -0,0 +1,70 @@
+name: test results
+
+# Post a clean check-run summary per test suite after the pytest workflow finishes.
+#
+# Replaces the older EnricoMi/publish-unit-test-result-action bot that posted busy
+# emoji-table PR comments accumulating across runs. dorny/test-reporter@v2 instead
+# posts a single well-formatted check run per suite with a table of passed / failed
+# tests, collapsible stack traces for failures, and source-line annotations. No
+# PR-comment spam; the result lives in the checks UI and on the "Details" page.
+#
+# Runs inside a workflow_run triggered by pytest so the downloaded JUnit XML
+# artifacts come with the write permissions needed to post check runs from
+# forked PRs (where the triggering workflow itself is read-only).
+
+on:
+  workflow_run:
+    workflows: ["pytest"]
+    types:
+      - completed
+
+permissions:
+  contents: read
+  checks: write
+  actions: read
+
+jobs:
+  test-results:
+    name: Test Results
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+
+    steps:
+      - name: Unit Tests report
+        uses: dorny/test-reporter@v2
+        with:
+          artifact: Unit Test Results
+          name: Unit Tests
+          path: "*.xml"
+          reporter: java-junit
+          fail-on-error: false
+          list-suites: failed
+          list-tests: failed
+          max-annotations: 50
+
+      - name: Integration Tests report
+        uses: dorny/test-reporter@v2
+        with:
+          # Six matrix jobs upload artifacts named
+          # "Integration Test Results (integration_tests_a)" .. _f. Match them all with
+          # a regex and aggregate into a single report.
+          artifact: /Integration Test Results .*/
+          name: Integration Tests
+          path: "*.xml"
+          reporter: java-junit
+          fail-on-error: false
+          list-suites: failed
+          list-tests: failed
+          max-annotations: 50
+
+      - name: Distributed Tests report
+        uses: dorny/test-reporter@v2
+        with:
+          artifact: /Distributed Test Results .*/
+          name: Distributed Tests
+          path: "*.xml"
+          reporter: java-junit
+          fail-on-error: false
+          list-suites: failed
+          list-tests: failed
+          max-annotations: 50

.github/workflows/upload-pypi.yml

@@ -0,0 +1,36 @@
+name: Upload to PyPI
+
+on:
+  # Triggers the workflow when a release or draft of a release is published,
+  # or a pre-release is changed to a release
+  release:
+    types: [released]
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  pypi-publish:
+    name: upload release to PyPI
+    runs-on: ubuntu-latest
+    # Specifying a GitHub environment is optional, but strongly encouraged
+    environment: release
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build source distribution and wheel
+        run: |
+          pip install build
+          python -m build
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1