mlflow--mlflow/tests/pyfunc/test_logged_models.py

import json
import os
from concurrent.futures import ThreadPoolExecutor

import pytest

import mlflow
from mlflow.entities.logged_model_status import LoggedModelStatus
from mlflow.exceptions import MlflowException
from mlflow.models import Model
from mlflow.tracing.constant import TraceMetadataKey
from mlflow.utils.mlflow_tags import MLFLOW_MODEL_IS_EXTERNAL


class DummyModel(mlflow.pyfunc.PythonModel):
    def predict(self, model_input):
        return len(model_input) * [0]


class TraceModel(mlflow.pyfunc.PythonModel):
    @mlflow.trace
    def predict(self, model_input):
        return len(model_input) * [0]


def test_model_id_tracking():
    model = TraceModel()
    model.predict([1, 2, 3])
    trace = mlflow.get_trace(mlflow.get_last_active_trace_id())
    assert TraceMetadataKey.MODEL_ID not in trace.info.request_metadata

    with mlflow.start_run():
        info = mlflow.pyfunc.log_model(name="my_model", python_model=model)
        # Log another model to ensure that the model ID is correctly associated with the first model
        mlflow.pyfunc.log_model(name="another_model", python_model=model)

    model = mlflow.pyfunc.load_model(info.model_uri)
    model.predict([4, 5, 6])

    trace = mlflow.get_trace(mlflow.get_last_active_trace_id())
    assert trace is not None
    assert trace.info.request_metadata[TraceMetadataKey.MODEL_ID] == info.model_id


def test_model_id_tracking_evaluate():
    with mlflow.start_run():
        info = mlflow.pyfunc.log_model(name="my_model", python_model=TraceModel())

    mlflow.evaluate(model=info.model_uri, data=[[1, 2, 3]], model_type="regressor", targets=[1])
    trace = mlflow.get_trace(mlflow.get_last_active_trace_id())
    assert trace is not None
    assert trace.info.request_metadata[TraceMetadataKey.MODEL_ID] == info.model_id


def test_model_id_tracking_thread_safety():
    models = []
    for _ in range(5):
        with mlflow.start_run():
            info = mlflow.pyfunc.log_model(
                name="my_model",
                python_model=TraceModel(),
                pip_requirements=[],  # to skip dependency inference
            )
            model = mlflow.pyfunc.load_model(info.model_uri)
            models.append(model)

    def predict(idx, model) -> None:
        model.predict([idx])

    with ThreadPoolExecutor(
        max_workers=len(models), thread_name_prefix="test-logged-models"
    ) as executor:
        futures = [executor.submit(predict, idx, model) for idx, model in enumerate(models)]
        for f in futures:
            f.result()

    traces = mlflow.search_traces(return_type="list")
    assert len(traces) == len(models)
    for trace in traces:
        trace_inputs = trace.info.request_metadata["mlflow.traceInputs"]
        index = json.loads(trace_inputs)["model_input"][0]
        model_id = trace.info.request_metadata["mlflow.modelId"]
        assert model_id == models[index].model_id


def test_run_params_are_logged_to_model():
    with mlflow.start_run():
        mlflow.log_params({"a": 1})
        mlflow.pyfunc.log_model(name="my_model", python_model=DummyModel())

    model = mlflow.last_logged_model()
    assert model.params == {"a": "1"}


def test_run_metrics_are_logged_to_model():
    with mlflow.start_run():
        mlflow.log_metrics({"a": 1, "b": 2})
        mlflow.pyfunc.log_model(name="my_model", python_model=DummyModel())

    model = mlflow.last_logged_model()
    assert [(m.key, m.value) for m in model.metrics] == [("a", 1), ("b", 2)]


def test_log_model_finalizes_existing_pending_model():
    model = mlflow.initialize_logged_model(name="testmodel")
    assert model.status == LoggedModelStatus.PENDING
    mlflow.pyfunc.log_model(python_model=DummyModel(), model_id=model.model_id)
    updated_model = mlflow.get_logged_model(model.model_id)
    assert updated_model.status == LoggedModelStatus.READY


def test_log_model_permits_logging_to_ready_model(tmp_path):
    # Create a non-external model and finalize it to READY status
    model = mlflow.initialize_logged_model(name="testmodel")
    model = mlflow.finalize_logged_model(model.model_id, LoggedModelStatus.READY)
    assert model.status == LoggedModelStatus.READY
    assert model.tags.get(MLFLOW_MODEL_IS_EXTERNAL, "false").lower() == "false"

    # Verify we can log to the READY model
    mlflow.pyfunc.log_model(python_model=DummyModel(), model_id=model.model_id)

    # Verify the model can be loaded
    mlflow.pyfunc.load_model(f"models:/{model.model_id}")

    # Verify the model artifacts were updated
    dst_dir = os.path.join(tmp_path, "dst")
    mlflow.artifacts.download_artifacts(f"models:/{model.model_id}", dst_path=dst_dir)
    mlflow_model = Model.load(os.path.join(dst_dir, "MLmodel"))
    assert mlflow_model.flavors.get("python_function") is not None


def test_log_model_permits_logging_model_artifacts_to_external_models(tmp_path):
    model = mlflow.create_external_model(name="testmodel")
    assert model.status == LoggedModelStatus.READY
    assert model.tags.get(MLFLOW_MODEL_IS_EXTERNAL) == "true"
    dst_dir_1 = os.path.join(tmp_path, "dst_1")
    mlflow.artifacts.download_artifacts(f"models:/{model.model_id}", dst_path=dst_dir_1)
    mlflow_model: Model = Model.load(os.path.join(dst_dir_1, "MLmodel"))

    model_info = mlflow.pyfunc.log_model(python_model=DummyModel(), model_id=model.model_id)

    # Verify that the model can now be loaded and is no longer tagged as external
    mlflow.pyfunc.load_model(model_info.model_uri)
    assert MLFLOW_MODEL_IS_EXTERNAL not in mlflow.get_logged_model(model.model_id).tags
    dst_dir_2 = os.path.join(tmp_path, "dst_2")
    mlflow.artifacts.download_artifacts(f"models:/{model.model_id}", dst_path=dst_dir_2)
    mlflow_model = Model.load(os.path.join(dst_dir_2, "MLmodel"))
    assert MLFLOW_MODEL_IS_EXTERNAL not in (mlflow_model.metadata or {})


def test_external_logged_model_cannot_be_loaded_with_pyfunc():
    model = mlflow.create_external_model(name="testmodel")
    with pytest.raises(
        MlflowException,
        match="This model's artifacts are external.*cannot be loaded",
    ):
        mlflow.pyfunc.load_model(f"models:/{model.model_id}")