name: MLflow tests

on:
  pull_request:
    types:
      - opened
      - synchronize
      - reopened
    paths-ignore:
      - "docs/**"
      - "**.md"
      - "dev/clint/**"
      - ".github/workflows/docs.yml"
      - ".github/workflows/preview-docs.yml"
      - "mlflow/server/js/**"
      - ".github/workflows/js.yml"
      - ".claude/**"
  push:
    branches:
      - master
      - branch-[0-9]+.[0-9]+

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

# Use `bash` by default for all `run` steps in this workflow:
# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#defaultsrun
defaults:
  run:
    shell: bash

env:
  MLFLOW_HOME: ${{ github.workspace }}
  # Note miniconda is pre-installed in the virtual environments for GitHub Actions:
  # https://github.com/actions/virtual-environments/blob/main/images/linux/scripts/installers/miniconda.sh
  MLFLOW_CONDA_HOME: /usr/share/miniconda
  SPARK_LOCAL_IP: localhost
  PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu
  PIP_CONSTRAINT: ${{ github.workspace }}/requirements/constraints.txt
  PYTHONUTF8: "1"
  _MLFLOW_TESTING_TELEMETRY: "true"
  MLFLOW_SERVER_ENABLE_JOB_EXECUTION: "false"

permissions: {}

jobs:
  # python-skinny tests cover a subset of mlflow functionality
  # that is meant to be supported with a smaller dependency footprint.
  # The python skinny tests cover the subset of mlflow functionality
  # while also verifying certain dependencies are omitted.
  python-skinny:
    if: github.event_name != 'pull_request' || (github.event.pull_request.draft == false || github.event.pull_request.user.login == 'Copilot' && github.event.pull_request.user.type == 'Bot')
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: ./.github/actions/untracked
      - uses: ./.github/actions/setup-python
      - name: Install dependencies
        run: |
          source ./dev/install-common-deps.sh --skinny
      - name: Run tests
        run: |
          ./dev/run-python-skinny-tests.sh

  python:
    if: github.event_name != 'pull_request' || (github.event.pull_request.draft == false || github.event.pull_request.user.login == 'Copilot' && github.event.pull_request.user.type == 'Bot')
    runs-on: ubuntu-latest
    timeout-minutes: 120
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        group: [1, 2, 3, 4]
        include:
          - splits: 4
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
          submodules: true
      - uses: ./.github/actions/untracked
      - uses: ./.github/actions/free-disk-space
      - uses: ./.github/actions/setup-python
      - uses: ./.github/actions/setup-pyenv
      - uses: ./.github/actions/setup-java
      - name: Install dependencies
        run: |
          uv sync --extra extras --extra gateway --extra mcp --extra genai
          uv pip install \
            -r requirements/test-requirements.txt \
            -r requirements/extra-ml-requirements.txt
      - uses: ./.github/actions/show-versions
      - uses: ./.github/actions/cache-hf
      - name: Import check
        run: |
          # `-I` is used to avoid importing modules from user-specific site-packages
          # that might conflict with the built-in modules (e.g. `types`).
          uv run --no-sync python -I tests/check_mlflow_lazily_imports_ml_packages.py
      - name: Run tests
        env:
          SPLITS: ${{ matrix.splits }}
          GROUP: ${{ matrix.group }}
        run: |
          source dev/setup-ssh.sh
          uv run --no-sync pytest --splits=$SPLITS --group=$GROUP \
            --quiet --requires-ssh --ignore-flavors \
            --ignore=tests/examples \
            --ignore=tests/evaluate \
            --ignore=tests/genai \
            --ignore=tests/docker \
            tests

      - name: Run databricks-connect related tests
        run: |
          # this needs to be run in a separate job because installing databricks-connect could break other
          # tests that uses normal SparkSession instead of remote SparkSession
          uv run --no-sync --with 'databricks-agents,databricks-connect' \
            pytest tests/utils/test_requirements_utils.py::test_infer_pip_requirements_on_databricks_agents

  database:
    if: github.event_name != 'pull_request' || (github.event.pull_request.draft == false || github.event.pull_request.user.login == 'Copilot' && github.event.pull_request.user.type == 'Bot')
    runs-on: ubuntu-latest
    timeout-minutes: 90
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: ./.github/actions/untracked
      - name: Build
        run: |
          ./tests/db/compose.sh pull -q postgresql mysql mssql
          docker images --digests
          ./tests/db/compose.sh build --build-arg DEPENDENCIES="$(python dev/extract_deps.py)"
      - name: Run tests
        run: |
          set +e
          err=0
          trap 'err=1' ERR
          RESULTS=""
          for service in $(./tests/db/compose.sh config --services | grep '^mlflow-' | sort)
          do
            # Set `--no-TTY` to show container logs on GitHub Actions:
            # https://github.com/actions/virtual-environments/issues/5022
            ./tests/db/compose.sh run --rm --no-TTY $service pytest \
              tests/store/tracking/sqlalchemy_store \
              tests/store/tracking/test_gateway_sql_store.py \
              tests/store/model_registry/test_sqlalchemy_store.py \
              tests/store/model_registry/test_sqlalchemy_workspace_store.py \
              tests/store/workspace/test_sqlalchemy_store.py \
              tests/db
              RESULTS="$RESULTS\n$service: $(if [ $? -eq 0 ]; then echo "✅"; else echo "❌"; fi)"
          done

          echo -e "$RESULTS"
          test $err = 0

      - name: Run migration check
        run: |
          set +e
          err=0
          trap 'err=1' ERR

          ./tests/db/compose.sh down --volumes --remove-orphans
          for service in $(./tests/db/compose.sh config --services | grep '^migration-')
          do
            ./tests/db/compose.sh run --rm --no-TTY $service
          done

          test $err = 0

      - name: Clean up
        run: |
          ./tests/db/compose.sh down --volumes --remove-orphans --rmi all

  java:
    if: github.event_name != 'pull_request' || (github.event.pull_request.draft == false || github.event.pull_request.user.login == 'Copilot' && github.event.pull_request.user.type == 'Bot')
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: ./.github/actions/untracked
      - uses: ./.github/actions/setup-python
      - uses: ./.github/actions/setup-pyenv
      - uses: ./.github/actions/setup-java
        with:
          java-version: 11
      - name: Install dependencies
        run: |
          uv sync --no-dev
      - name: Run tests
        working-directory: mlflow/java
        run: |
          uv run --no-dev mvn clean package -q

  flavors:
    if: github.event_name != 'pull_request' || (github.event.pull_request.draft == false || github.event.pull_request.user.login == 'Copilot' && github.event.pull_request.user.type == 'Bot')
    runs-on: ubuntu-latest
    timeout-minutes: 120
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: ./.github/actions/untracked
      - uses: ./.github/actions/setup-python
      - uses: ./.github/actions/setup-pyenv
      - uses: ./.github/actions/setup-java
      - name: Install dependencies
        run: |
          uv sync --extra extras
          uv pip install \
            -r requirements/test-requirements.txt \
            -r requirements/extra-ml-requirements.txt \
            'tensorflow<2.21'
      - uses: ./.github/actions/show-versions
      - name: Run tests
        run: |
          uv run --no-sync pytest \
            tests/tracking/fluent/test_fluent_autolog.py \
            tests/autologging

  models:
    if: github.event_name != 'pull_request' || (github.event.pull_request.draft == false || github.event.pull_request.user.login == 'Copilot' && github.event.pull_request.user.type == 'Bot')
    runs-on: ubuntu-latest
    timeout-minutes: 120
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        group: [1, 2, 3]
        include:
          - splits: 3
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: ./.github/actions/untracked
      - uses: ./.github/actions/free-disk-space
      - uses: ./.github/actions/setup-python
      - uses: ./.github/actions/setup-pyenv
      - uses: ./.github/actions/setup-java
      - name: Install dependencies
        run: |
          uv sync
          uv pip install \
            -r requirements/test-requirements.txt \
            pyspark langchain langchain-community
      - uses: ./.github/actions/show-versions
      - name: Run tests
        env:
          SPLITS: ${{ matrix.splits }}
          GROUP: ${{ matrix.group }}
        run: |
          uv run --no-sync pytest --splits=$SPLITS --group=$GROUP tests/models

  evaluate:
    if: github.event_name != 'pull_request' || (github.event.pull_request.draft == false || github.event.pull_request.user.login == 'Copilot' && github.event.pull_request.user.type == 'Bot')
    runs-on: ubuntu-latest
    timeout-minutes: 90
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        group: [1]
        include:
          - splits: 1
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: ./.github/actions/untracked
      - uses: ./.github/actions/setup-python
      - uses: ./.github/actions/setup-pyenv
      - uses: ./.github/actions/setup-java
      - name: Install dependencies
        run: |
          uv sync --extra extras --extra genai
          uv pip install \
            -r requirements/test-requirements.txt \
            torch transformers pyspark langchain langchain-experimental shap lightgbm xgboost
      - uses: ./.github/actions/show-versions
      - name: Run tests
        env:
          SPLITS: ${{ matrix.splits }}
          GROUP: ${{ matrix.group }}
        run: |
          uv run --no-sync pytest --splits=$SPLITS --group=$GROUP tests/evaluate --ignore=tests/evaluate/test_default_evaluator_delta.py
      - name: Run tests with delta
        run: |
          uv run --no-sync pytest tests/evaluate/test_default_evaluator_delta.py

  genai:
    if: github.event_name != 'pull_request' || (github.event.pull_request.draft == false || github.event.pull_request.user.login == 'Copilot' && github.event.pull_request.user.type == 'Bot')
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: ./.github/actions/untracked
      - uses: ./.github/actions/setup-python
      - uses: ./.github/actions/setup-pyenv
      - uses: ./.github/actions/setup-java
      - name: Install dependencies
        run: |
          uv sync --extra genai
          # TODO: migrate mlflow/genai/scorers/phoenix to arize-phoenix-evals 3.x API
          # (removed HallucinationEvaluator/RelevanceEvaluator/etc. and LiteLLMModel)
          # TODO: drop the langchain-community<0.4.2 pin once ragas releases a
          # fix for the removed `langchain_community.chat_models.vertexai`
          # import (0.4.2 dropped the shim).
          # https://github.com/vibrantlabsai/ragas/issues/2745
          #
          # guardrails-ai is intentionally omitted: it caps typer<0.20 and
          # click<=8.2.0, which the synced env no longer satisfies, so an
          # unpinned install backtracks to the ancient 0.1.8 (imports the
          # removed openai.error). Restore it once upstream lifts the pins.
          # https://github.com/guardrails-ai/guardrails/issues/1505
          uv pip install \
            -r requirements/test-requirements.txt \
            deepeval ragas 'arize-phoenix-evals<3.0.0' 'langchain-community<0.4.2' trulens trulens-providers-litellm 'google-adk[gcp]'
      - uses: ./.github/actions/show-versions
      - name: Run GenAI Tests (OSS)
        run: |
          # guardrails-ai is not installed (see the install step), so skip its
          # tests to keep collection of the rest of tests/genai from breaking.
          # https://github.com/guardrails-ai/guardrails/issues/1505
          uv run --no-sync pytest tests/genai \
            --ignore tests/genai/scorers/guardrails

      - name: Run GenAI Tests (Databricks)
        run: |
          # guardrails-ai is not installed (see the install step), so skip its
          # tests here too. https://github.com/guardrails-ai/guardrails/issues/1505
          #
          # test_pytest_integration.py spawns a subprocess pytest from a temp dir,
          # which imports the published mlflow-skinny that databricks-agents pulls
          # into this overlay rather than the local checkout. That published build
          # lacks the new `mlflow.test` API, so the generated test fails to collect.
          # It already runs against the local mlflow in the OSS step above.
          uv run --no-sync --with databricks-agents \
            pytest tests/genai --ignore tests/genai/test_genai_import_without_agent_sdk.py \
            --ignore tests/genai/optimize --ignore tests/genai/prompts \
            --ignore tests/genai/scorers/guardrails \
            --ignore tests/genai/evaluate/test_pytest_integration.py

  pyfunc:
    if: github.event_name != 'pull_request' || (github.event.pull_request.draft == false || github.event.pull_request.user.login == 'Copilot' && github.event.pull_request.user.type == 'Bot')
    runs-on: ubuntu-latest
    timeout-minutes: 120
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        group: [1, 2, 3, 4]
        include:
          - splits: 4
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: ./.github/actions/untracked
      - uses: ./.github/actions/free-disk-space
      - uses: ./.github/actions/setup-python
      - uses: ./.github/actions/setup-pyenv
      - uses: ./.github/actions/setup-java
      - name: Install dependencies
        run: |
          uv sync --extra extras --extra gateway
          uv pip install \
            -r requirements/test-requirements.txt \
            tensorflow 'pyspark[connect]'
      - uses: ./.github/actions/show-versions
      - name: Run tests
        env:
          SPLITS: ${{ matrix.splits }}
          GROUP: ${{ matrix.group }}
        run: |
          uv run --no-sync pytest --splits=$SPLITS --group=$GROUP --durations=30 \
            tests/pyfunc tests/types --ignore tests/pyfunc/test_spark_connect.py

          # test_spark_connect.py fails if it's run with other tests, so run it separately.
          uv run --no-sync pytest tests/pyfunc/test_spark_connect.py

  windows:
    if: github.event_name != 'pull_request' || (github.event.pull_request.draft == false || github.event.pull_request.user.login == 'Copilot' && github.event.pull_request.user.type == 'Bot')
    runs-on: windows-latest
    timeout-minutes: 120
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        group: [1, 2, 3]
        include:
          - splits: 3
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
          submodules: true
      - uses: ./.github/actions/untracked
      - uses: ./.github/actions/setup-python
      - uses: ./.github/actions/setup-pyenv
      - uses: ./.github/actions/setup-java
      - name: Install python dependencies
        run: |
          uv sync --extra extras --extra genai --extra mcp
          uv pip install \
            -r requirements/test-requirements.txt \
            pyspark datasets tensorflow torch transformers openai \
            tests/resources/mlflow-test-plugin
      - uses: ./.github/actions/show-versions
      - uses: ./.github/actions/cache-hf
      - name: Download Hadoop winutils for Spark
        run: |
          git clone https://github.com/cdarlint/winutils /tmp/winutils
      - name: Run python tests
        env:
          # Set Hadoop environment variables required for testing Spark integrations on Windows
          HADOOP_HOME: /tmp/winutils/hadoop-3.2.2
          # Use non-GUI matplotlib backend since Tkinter may not be installed properly on Windows runners
          # https://github.com/stefmolin/data-morph/pull/273
          MPLBACKEND: Agg
          SPLITS: ${{ matrix.splits }}
          GROUP: ${{ matrix.group }}
        run: |
          export PATH=$PATH:$HADOOP_HOME/bin
          uv run --no-sync pytest tests \
            --splits=$SPLITS \
            --group=$GROUP \
            --ignore-flavors \
            --ignore=tests/projects \
            --ignore=tests/examples \
            --ignore=tests/evaluate \
            --ignore=tests/optuna \
            --ignore=tests/pyspark/optuna \
            --ignore=tests/genai \
            --ignore=tests/sagemaker \
            --ignore=tests/gateway \
            --ignore=tests/server/auth \
            --ignore=tests/data/test_spark_dataset.py \
            --ignore=tests/data/test_spark_dataset_source.py \
            --ignore=tests/data/test_delta_dataset_source.py \
            --deselect=tests/data/test_pandas_dataset.py::test_from_pandas_spark_datasource \
            --deselect=tests/data/test_pandas_dataset.py::test_from_pandas_delta_datasource \
            --ignore=tests/docker \
            --ignore=tests/demo