wehub-resource-sync
1812 lines
79 KiB
Python
1812 lines
79 KiB
Python
import inspect
|
|
import json
|
|
import keyword
|
|
import logging
|
|
import os
|
|
import pathlib
|
|
import signal
|
|
import urllib.parse
|
|
from abc import ABCMeta, abstractmethod
|
|
from contextlib import contextmanager, nullcontext
|
|
from dataclasses import dataclass
|
|
from inspect import Parameter, Signature
|
|
from types import FunctionType
|
|
from typing import Any
|
|
|
|
import mlflow
|
|
from mlflow.data.dataset import Dataset
|
|
from mlflow.data.evaluation_dataset import (
|
|
EvaluationDataset,
|
|
convert_data_to_mlflow_dataset,
|
|
)
|
|
from mlflow.entities.dataset_input import DatasetInput
|
|
from mlflow.entities.input_tag import InputTag
|
|
from mlflow.entities.logged_model_input import LoggedModelInput
|
|
from mlflow.exceptions import MlflowException
|
|
from mlflow.models.evaluation.utils.trace import configure_autologging_for_evaluation
|
|
from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE
|
|
from mlflow.store.artifact.utils.models import _parse_model_id_if_present
|
|
from mlflow.telemetry.events import EvaluateEvent
|
|
from mlflow.telemetry.track import record_usage_event
|
|
from mlflow.tracking.artifact_utils import _download_artifact_from_uri
|
|
from mlflow.tracking.client import MlflowClient
|
|
from mlflow.tracking.fluent import _set_active_model
|
|
from mlflow.utils import _get_fully_qualified_class_name
|
|
from mlflow.utils.annotations import developer_stable
|
|
from mlflow.utils.class_utils import _get_class_from_string
|
|
from mlflow.utils.file_utils import TempDir
|
|
from mlflow.utils.mlflow_tags import MLFLOW_DATASET_CONTEXT
|
|
from mlflow.utils.proto_json_utils import NumpyEncoder
|
|
|
|
try:
|
|
# `pandas` is not required for `mlflow-skinny`.
|
|
import pandas as pd
|
|
except ImportError:
|
|
pass
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
|
|
class _ModelType:
|
|
REGRESSOR = "regressor"
|
|
CLASSIFIER = "classifier"
|
|
QUESTION_ANSWERING = "question-answering"
|
|
TEXT_SUMMARIZATION = "text-summarization"
|
|
TEXT = "text"
|
|
RETRIEVER = "retriever"
|
|
# This model type is used for Mosaic AI Agent evaluation and only available in Databricks
|
|
# https://docs.databricks.com/en/generative-ai/agent-evaluation/index.html
|
|
DATABRICKS_AGENT = "databricks-agent"
|
|
|
|
def __init__(self):
|
|
raise NotImplementedError("This class is not meant to be instantiated.")
|
|
|
|
@classmethod
|
|
def values(cls):
|
|
return (
|
|
cls.REGRESSOR,
|
|
cls.CLASSIFIER,
|
|
cls.QUESTION_ANSWERING,
|
|
cls.TEXT_SUMMARIZATION,
|
|
cls.TEXT,
|
|
cls.RETRIEVER,
|
|
)
|
|
|
|
|
|
class EvaluationMetric:
|
|
'''
|
|
An evaluation metric.
|
|
|
|
Args:
|
|
eval_fn: A function that computes the metric with the following signature:
|
|
|
|
.. code-block:: python
|
|
|
|
def eval_fn(
|
|
predictions: pandas.Series,
|
|
targets: pandas.Series,
|
|
metrics: Dict[str, MetricValue],
|
|
**kwargs,
|
|
) -> Union[float, MetricValue]:
|
|
"""
|
|
Args:
|
|
predictions: A pandas Series containing the predictions made by the model.
|
|
targets: (Optional) A pandas Series containing the corresponding labels
|
|
for the predictions made on that input.
|
|
metrics: (Optional) A dictionary containing the metrics calculated by the
|
|
default evaluator. The keys are the names of the metrics and the values
|
|
are the metric values. To access the MetricValue for the metrics
|
|
calculated by the system, make sure to specify the type hint for this
|
|
parameter as Dict[str, MetricValue]. Refer to the DefaultEvaluator
|
|
behavior section for what metrics will be returned based on the type of
|
|
model (i.e. classifier or regressor).
|
|
kwargs: Includes a list of args that are used to compute the metric. These
|
|
args could be information coming from input data, model outputs,
|
|
other metrics, or parameters specified in the `evaluator_config`
|
|
argument of the `mlflow.evaluate` API.
|
|
|
|
Returns: MetricValue with per-row scores, per-row justifications, and aggregate
|
|
results.
|
|
"""
|
|
...
|
|
|
|
name: The name of the metric.
|
|
greater_is_better: Whether a greater value of the metric is better.
|
|
long_name: (Optional) The long name of the metric. For example,
|
|
``"root_mean_squared_error"`` for ``"mse"``.
|
|
version: (Optional) The metric version. For example ``v1``.
|
|
metric_details: (Optional) A description of the metric and how it is calculated.
|
|
metric_metadata: (Optional) A dictionary containing metadata for the metric.
|
|
genai_metric_args: (Optional) A dictionary containing arguments specified by users
|
|
when calling make_genai_metric or make_genai_metric_from_prompt. Those args
|
|
are persisted so that we can deserialize the same metric object later.
|
|
'''
|
|
|
|
def __init__(
|
|
self,
|
|
eval_fn,
|
|
name,
|
|
greater_is_better,
|
|
long_name=None,
|
|
version=None,
|
|
metric_details=None,
|
|
metric_metadata=None,
|
|
genai_metric_args=None,
|
|
):
|
|
self.eval_fn = eval_fn
|
|
self.name = name
|
|
self.greater_is_better = greater_is_better
|
|
self.long_name = long_name or name
|
|
self.version = version
|
|
self.metric_details = metric_details
|
|
self.metric_metadata = metric_metadata
|
|
self.genai_metric_args = genai_metric_args
|
|
|
|
def __str__(self):
|
|
parts = [f"name={self.name}, greater_is_better={self.greater_is_better}"]
|
|
|
|
if self.long_name:
|
|
parts.append(f"long_name={self.long_name}")
|
|
if self.version:
|
|
parts.append(f"version={self.version}")
|
|
if self.metric_details:
|
|
parts.append(f"metric_details={self.metric_details}")
|
|
if self.metric_metadata:
|
|
parts.append(f"metric_metadata={self.metric_metadata}")
|
|
|
|
return "EvaluationMetric(" + ", ".join(parts) + ")"
|
|
|
|
|
|
# NB: we need this function because we cannot modify the signature of
|
|
# a class's __call__ method after the class has been defined.
|
|
# This is also useful to distinguish between the metric signatures with different eval_fn signatures
|
|
def _generate_eval_metric_class(eval_fn, require_strict_signature=False):
|
|
"""
|
|
Dynamically generate a GenAIEvaluationMetric class that can be used to evaluate the metric
|
|
on the given input data. The generated class is callable with a __call__ method that
|
|
takes the arguments specified in the signature of the eval_fn function.
|
|
|
|
Args:
|
|
eval_fn: the evaluation function of the EvaluationMetric.
|
|
require_strict_signature: (Optional) Whether the eval_fn needs to follow a strict signature.
|
|
If True, then the eval_fn must follow below signature:
|
|
|
|
.. code-block:: python
|
|
|
|
def eval_fn(
|
|
predictions: "pd.Series",
|
|
metrics: Dict[str, MetricValue],
|
|
inputs: "pd.Series",
|
|
*args,
|
|
) -> MetricValue:
|
|
pass
|
|
|
|
When generating a metric from `make_genai_metric`, this should be set to True.
|
|
Default to False.
|
|
|
|
Returns:
|
|
A dynamically generated callable CallableEvaluationMetric class.
|
|
"""
|
|
from mlflow.metrics.base import MetricValue
|
|
|
|
if require_strict_signature:
|
|
allowed_kwargs_names = [
|
|
param_name
|
|
for param_name in inspect.signature(eval_fn).parameters.keys()
|
|
if param_name not in ["predictions", "metrics", "inputs"]
|
|
]
|
|
|
|
def genai_call_method(
|
|
self,
|
|
*,
|
|
predictions: pd.Series | str | list[str],
|
|
inputs: pd.Series | str | list[str],
|
|
metrics: dict[str, MetricValue] | None = None,
|
|
**kwargs,
|
|
) -> MetricValue:
|
|
if missed_kwargs := set(allowed_kwargs_names) - set(kwargs.keys()):
|
|
raise MlflowException.invalid_parameter_value(
|
|
f"Missing required arguments: {missed_kwargs}",
|
|
)
|
|
if extra_kwargs := set(kwargs.keys()) - set(allowed_kwargs_names):
|
|
raise MlflowException.invalid_parameter_value(
|
|
f"Unexpected arguments: {extra_kwargs}",
|
|
)
|
|
return self.eval_fn(
|
|
_convert_val_to_pd_Series(predictions, "predictions"),
|
|
metrics or {},
|
|
_convert_val_to_pd_Series(inputs, "inputs"),
|
|
# Note: based on https://github.com/mlflow/mlflow/blob/4fef77afdbe4d76302cb0b1aad2bd72b5cde64e9/mlflow/metrics/genai/genai_metric.py#L49-L53
|
|
# the extra params passed https://github.com/mlflow/mlflow/blob/4fef77afdbe4d76302cb0b1aad2bd72b5cde64e9/mlflow/metrics/genai/genai_metric.py#L513
|
|
# should always be pandas Series
|
|
*[
|
|
_convert_val_to_pd_Series(kwargs[arg_name], arg_name)
|
|
for arg_name in allowed_kwargs_names
|
|
],
|
|
)
|
|
|
|
genai_call_method.__signature__ = Signature(
|
|
parameters=[
|
|
Parameter("self", Parameter.POSITIONAL_OR_KEYWORD),
|
|
Parameter(
|
|
"predictions",
|
|
Parameter.KEYWORD_ONLY,
|
|
annotation=pd.Series | str | list[str],
|
|
),
|
|
Parameter(
|
|
"inputs",
|
|
Parameter.KEYWORD_ONLY,
|
|
annotation=pd.Series | str | list[str],
|
|
),
|
|
Parameter(
|
|
"metrics",
|
|
Parameter.KEYWORD_ONLY,
|
|
annotation=dict[str, MetricValue] | None,
|
|
default=None,
|
|
),
|
|
*[
|
|
Parameter(name, Parameter.KEYWORD_ONLY, annotation=pd.Series | str | list[str])
|
|
for name in allowed_kwargs_names
|
|
],
|
|
]
|
|
)
|
|
genai_call_method.__doc__ = f"""
|
|
Evaluate the metric on the given inputs and predictions.
|
|
Note: only keyword arguments are supported.
|
|
|
|
Args:
|
|
predictions: predictions made by the model.
|
|
inputs: inputs used to make the predictions.
|
|
metrics: metrics calculated by the default evaluator.
|
|
kwargs: additional arguments used to compute the metric.
|
|
Required arguments: {allowed_kwargs_names}
|
|
|
|
Returns:
|
|
evaluation result as MetricValue object.
|
|
"""
|
|
call_method = genai_call_method
|
|
|
|
else:
|
|
|
|
def _call_method(
|
|
self,
|
|
**kwargs,
|
|
) -> MetricValue:
|
|
return self.eval_fn(**kwargs)
|
|
|
|
allowed_kwargs_params = inspect.signature(eval_fn).parameters
|
|
_call_method.__signature__ = Signature(
|
|
parameters=[
|
|
Parameter("self", Parameter.POSITIONAL_OR_KEYWORD),
|
|
*[
|
|
Parameter(
|
|
name,
|
|
Parameter.KEYWORD_ONLY,
|
|
annotation=allowed_kwargs_params[name].annotation,
|
|
)
|
|
for name in allowed_kwargs_params.keys()
|
|
],
|
|
]
|
|
)
|
|
_call_method.__doc__ = f"""
|
|
Evaluate the metric on the given inputs and predictions.
|
|
Note: only keyword arguments are supported.
|
|
|
|
Args:
|
|
kwargs: additional arguments used to compute the metric.
|
|
Required arguments: {list(allowed_kwargs_params.keys())}
|
|
|
|
Returns:
|
|
evaluation result as MetricValue object.
|
|
"""
|
|
call_method = _call_method
|
|
|
|
return type(
|
|
"CallableEvaluationMetric",
|
|
(EvaluationMetric,),
|
|
{"__call__": call_method},
|
|
)
|
|
|
|
|
|
def _convert_val_to_pd_Series(val, name):
|
|
if val is not None and not isinstance(val, pd.Series):
|
|
if isinstance(val, str):
|
|
return pd.Series([val])
|
|
elif isinstance(val, list):
|
|
return pd.Series(val)
|
|
else:
|
|
raise TypeError(
|
|
f"Expected {name} to be a string, list, or Pandas Series, got {type(val)}"
|
|
)
|
|
return val
|
|
|
|
|
|
def make_metric(
|
|
*,
|
|
eval_fn,
|
|
greater_is_better,
|
|
name=None,
|
|
long_name=None,
|
|
version=None,
|
|
metric_details=None,
|
|
metric_metadata=None,
|
|
genai_metric_args=None,
|
|
):
|
|
'''
|
|
A factory function to create an :py:class:`EvaluationMetric` object.
|
|
|
|
Args:
|
|
eval_fn: A function that computes the metric with the following signature:
|
|
|
|
.. code-block:: python
|
|
|
|
def eval_fn(
|
|
predictions: pandas.Series,
|
|
targets: pandas.Series,
|
|
metrics: Dict[str, MetricValue],
|
|
**kwargs,
|
|
) -> Union[float, MetricValue]:
|
|
"""
|
|
Args:
|
|
predictions: A pandas Series containing the predictions made by the model.
|
|
targets: (Optional) A pandas Series containing the corresponding labels
|
|
for the predictions made on that input.
|
|
metrics: (Optional) A dictionary containing the metrics calculated by the
|
|
default evaluator. The keys are the names of the metrics and the values
|
|
are the metric values. To access the MetricValue for the metrics
|
|
calculated by the system, make sure to specify the type hint for this
|
|
parameter as Dict[str, MetricValue]. Refer to the DefaultEvaluator
|
|
behavior section for what metrics will be returned based on the type of
|
|
model (i.e. classifier or regressor). kwargs: Includes a list of args
|
|
that are used to compute the metric. These args could information coming
|
|
from input data, model outputs or parameters specified in the
|
|
`evaluator_config` argument of the `mlflow.evaluate` API.
|
|
kwargs: Includes a list of args that are used to compute the metric. These
|
|
args could be information coming from input data, model outputs,
|
|
other metrics, or parameters specified in the `evaluator_config`
|
|
argument of the `mlflow.evaluate` API.
|
|
|
|
Returns: MetricValue with per-row scores, per-row justifications, and aggregate
|
|
results.
|
|
"""
|
|
...
|
|
|
|
greater_is_better: Whether a greater value of the metric is better.
|
|
name: The name of the metric. This argument must be specified if ``eval_fn`` is a lambda
|
|
function or the ``eval_fn.__name__`` attribute is not available.
|
|
long_name: (Optional) The long name of the metric. For example, ``"mean_squared_error"``
|
|
for ``"mse"``.
|
|
version: (Optional) The metric version. For example ``v1``.
|
|
metric_details: (Optional) A description of the metric and how it is calculated.
|
|
metric_metadata: (Optional) A dictionary containing metadata for the metric.
|
|
genai_metric_args: (Optional) A dictionary containing arguments specified by users
|
|
when calling make_genai_metric or make_genai_metric_from_prompt. Those args
|
|
are persisted so that we can deserialize the same metric object later.
|
|
|
|
.. seealso::
|
|
|
|
- :py:class:`mlflow.models.EvaluationMetric`
|
|
- :py:func:`mlflow.evaluate`
|
|
'''
|
|
return _make_metric(
|
|
eval_fn=eval_fn,
|
|
greater_is_better=greater_is_better,
|
|
name=name,
|
|
long_name=long_name,
|
|
version=version,
|
|
metric_details=metric_details,
|
|
metric_metadata=metric_metadata,
|
|
genai_metric_args=genai_metric_args,
|
|
require_strict_signature=False,
|
|
)
|
|
|
|
|
|
def _make_metric(
|
|
*,
|
|
eval_fn,
|
|
greater_is_better,
|
|
name=None,
|
|
long_name=None,
|
|
version=None,
|
|
metric_details=None,
|
|
metric_metadata=None,
|
|
genai_metric_args=None,
|
|
require_strict_signature=False,
|
|
):
|
|
'''
|
|
A factory function to create an :py:class:`EvaluationMetric` object.
|
|
|
|
Args:
|
|
eval_fn: A function that computes the metric with the following signature:
|
|
|
|
.. code-block:: python
|
|
|
|
def eval_fn(
|
|
predictions: pandas.Series,
|
|
targets: pandas.Series,
|
|
metrics: Dict[str, MetricValue],
|
|
**kwargs,
|
|
) -> Union[float, MetricValue]:
|
|
"""
|
|
Args:
|
|
predictions: A pandas Series containing the predictions made by the model.
|
|
targets: (Optional) A pandas Series containing the corresponding labels
|
|
for the predictions made on that input.
|
|
metrics: (Optional) A dictionary containing the metrics calculated by the
|
|
default evaluator. The keys are the names of the metrics and the values
|
|
are the metric values. To access the MetricValue for the metrics
|
|
calculated by the system, make sure to specify the type hint for this
|
|
parameter as Dict[str, MetricValue]. Refer to the DefaultEvaluator
|
|
behavior section for what metrics will be returned based on the type of
|
|
model (i.e. classifier or regressor). kwargs: Includes a list of args
|
|
that are used to compute the metric. These args could information coming
|
|
from input data, model outputs or parameters specified in the
|
|
`evaluator_config` argument of the `mlflow.evaluate` API.
|
|
kwargs: Includes a list of args that are used to compute the metric. These
|
|
args could be information coming from input data, model outputs,
|
|
other metrics, or parameters specified in the `evaluator_config`
|
|
argument of the `mlflow.evaluate` API.
|
|
|
|
Returns: MetricValue with per-row scores, per-row justifications, and aggregate
|
|
results.
|
|
"""
|
|
...
|
|
|
|
greater_is_better: Whether a greater value of the metric is better.
|
|
name: The name of the metric. This argument must be specified if ``eval_fn`` is a lambda
|
|
function or the ``eval_fn.__name__`` attribute is not available.
|
|
long_name: (Optional) The long name of the metric. For example, ``"mean_squared_error"``
|
|
for ``"mse"``.
|
|
version: (Optional) The metric version. For example ``v1``.
|
|
metric_details: (Optional) A description of the metric and how it is calculated.
|
|
metric_metadata: (Optional) A dictionary containing metadata for the metric.
|
|
genai_metric_args: (Optional) A dictionary containing arguments specified by users
|
|
when calling make_genai_metric or make_genai_metric_from_prompt. Those args
|
|
are persisted so that we can deserialize the same metric object later.
|
|
require_strict_signature: (Optional) Whether the eval_fn needs to follow a strict signature.
|
|
If True, then the eval_fn must follow below signature:
|
|
|
|
.. code-block:: python
|
|
|
|
def eval_fn(
|
|
predictions: "pd.Series",
|
|
metrics: Dict[str, MetricValue],
|
|
inputs: "pd.Series",
|
|
*args,
|
|
) -> MetricValue:
|
|
pass
|
|
|
|
When generating a metric from `make_genai_metric`, this should be set to True.
|
|
Default to False.
|
|
|
|
.. seealso::
|
|
|
|
- :py:class:`mlflow.models.EvaluationMetric`
|
|
- :py:func:`mlflow.evaluate`
|
|
'''
|
|
if name is None:
|
|
if isinstance(eval_fn, FunctionType) and eval_fn.__name__ == "<lambda>":
|
|
raise MlflowException(
|
|
"`name` must be specified if `eval_fn` is a lambda function.",
|
|
INVALID_PARAMETER_VALUE,
|
|
)
|
|
if not hasattr(eval_fn, "__name__"):
|
|
raise MlflowException(
|
|
"`name` must be specified if `eval_fn` does not have a `__name__` attribute.",
|
|
INVALID_PARAMETER_VALUE,
|
|
)
|
|
name = eval_fn.__name__
|
|
|
|
if "/" in name:
|
|
raise MlflowException(
|
|
f"Invalid metric name '{name}'. Metric names cannot include forward slashes ('/').",
|
|
INVALID_PARAMETER_VALUE,
|
|
)
|
|
|
|
if not name.isidentifier():
|
|
_logger.warning(
|
|
f"The metric name '{name}' provided is not a valid Python identifier, which will "
|
|
"prevent its use as a base metric for derived metrics. Please use a valid identifier "
|
|
"to enable creation of derived metrics that use the given metric."
|
|
)
|
|
|
|
if keyword.iskeyword(name):
|
|
_logger.warning(
|
|
f"The metric name '{name}' is a reserved Python keyword, which will "
|
|
"prevent its use as a base metric for derived metrics. Please use a valid identifier "
|
|
"to enable creation of derived metrics that use the given metric."
|
|
)
|
|
|
|
if name in ["predictions", "targets", "metrics"]:
|
|
_logger.warning(
|
|
f"The metric name '{name}' is used as a special parameter in MLflow metrics, which "
|
|
"will prevent its use as a base metric for derived metrics. Please use a different "
|
|
"name to enable creation of derived metrics that use the given metric."
|
|
)
|
|
|
|
return _generate_eval_metric_class(eval_fn, require_strict_signature=require_strict_signature)(
|
|
eval_fn=eval_fn,
|
|
name=name,
|
|
greater_is_better=greater_is_better,
|
|
long_name=long_name,
|
|
version=version,
|
|
metric_details=metric_details,
|
|
metric_metadata=metric_metadata,
|
|
genai_metric_args=genai_metric_args,
|
|
)
|
|
|
|
|
|
@developer_stable
|
|
class EvaluationArtifact(metaclass=ABCMeta):
|
|
"""
|
|
A model evaluation artifact containing an artifact uri and content.
|
|
"""
|
|
|
|
def __init__(self, uri, content=None):
|
|
self._uri = uri
|
|
self._content = content
|
|
|
|
@abstractmethod
|
|
def _load_content_from_file(self, local_artifact_path):
|
|
"""
|
|
Abstract interface to load the content from local artifact file path,
|
|
and return the loaded content.
|
|
"""
|
|
|
|
def _load(self, local_artifact_path=None):
|
|
"""
|
|
If ``local_artifact_path`` is ``None``, download artifact from the artifact uri.
|
|
Otherwise, load artifact content from the specified path. Assign the loaded content to
|
|
``self._content``, and return the loaded content.
|
|
"""
|
|
if local_artifact_path is not None:
|
|
self._content = self._load_content_from_file(local_artifact_path)
|
|
else:
|
|
with TempDir() as temp_dir:
|
|
temp_dir_path = temp_dir.path()
|
|
_download_artifact_from_uri(self._uri, temp_dir_path)
|
|
local_artifact_file = temp_dir.path(os.listdir(temp_dir_path)[0])
|
|
self._content = self._load_content_from_file(local_artifact_file)
|
|
return self._content
|
|
|
|
@abstractmethod
|
|
def _save(self, output_artifact_path):
|
|
"""Save artifact content into specified path."""
|
|
|
|
@property
|
|
def content(self):
|
|
"""
|
|
The content of the artifact (representation varies)
|
|
"""
|
|
if self._content is None:
|
|
self._load()
|
|
return self._content
|
|
|
|
@property
|
|
def uri(self) -> str:
|
|
"""
|
|
The URI of the artifact
|
|
"""
|
|
return self._uri
|
|
|
|
def __repr__(self):
|
|
return f"{self.__class__.__name__}(uri='{self.uri}')"
|
|
|
|
|
|
class EvaluationResult:
|
|
"""
|
|
Represents the model evaluation outputs of a `mlflow.evaluate()` API call, containing
|
|
both scalar metrics and output artifacts such as performance plots.
|
|
"""
|
|
|
|
def __init__(self, metrics, artifacts, run_id=None):
|
|
self._metrics = metrics
|
|
self._artifacts = artifacts
|
|
self._run_id = (
|
|
run_id
|
|
if run_id is not None
|
|
else (mlflow.active_run().info.run_id if mlflow.active_run() is not None else None)
|
|
)
|
|
|
|
@classmethod
|
|
def load(cls, path):
|
|
"""Load the evaluation results from the specified local filesystem path"""
|
|
with open(os.path.join(path, "metrics.json")) as fp:
|
|
metrics = json.load(fp)
|
|
|
|
with open(os.path.join(path, "artifacts_metadata.json")) as fp:
|
|
artifacts_metadata = json.load(fp)
|
|
|
|
artifacts = {}
|
|
|
|
artifacts_dir = os.path.join(path, "artifacts")
|
|
|
|
for artifact_name, meta in artifacts_metadata.items():
|
|
uri = meta["uri"]
|
|
ArtifactCls = _get_class_from_string(meta["class_name"])
|
|
artifact = ArtifactCls(uri=uri)
|
|
filename = pathlib.Path(urllib.parse.urlparse(uri).path).name
|
|
artifact._load(os.path.join(artifacts_dir, filename))
|
|
artifacts[artifact_name] = artifact
|
|
|
|
return EvaluationResult(metrics=metrics, artifacts=artifacts)
|
|
|
|
def save(self, path):
|
|
"""Write the evaluation results to the specified local filesystem path"""
|
|
os.makedirs(path, exist_ok=True)
|
|
with open(os.path.join(path, "metrics.json"), "w") as fp:
|
|
json.dump(self.metrics, fp, cls=NumpyEncoder)
|
|
|
|
artifacts_metadata = {
|
|
artifact_name: {
|
|
"uri": artifact.uri,
|
|
"class_name": _get_fully_qualified_class_name(artifact),
|
|
}
|
|
for artifact_name, artifact in self.artifacts.items()
|
|
}
|
|
with open(os.path.join(path, "artifacts_metadata.json"), "w") as fp:
|
|
json.dump(artifacts_metadata, fp)
|
|
|
|
artifacts_dir = os.path.join(path, "artifacts")
|
|
os.makedirs(artifacts_dir, exist_ok=True)
|
|
|
|
for artifact in self.artifacts.values():
|
|
filename = pathlib.Path(urllib.parse.urlparse(artifact.uri).path).name
|
|
artifact._save(os.path.join(artifacts_dir, filename))
|
|
|
|
@property
|
|
def metrics(self) -> dict[str, Any]:
|
|
"""
|
|
A dictionary mapping scalar metric names to scalar metric values
|
|
"""
|
|
return self._metrics
|
|
|
|
@property
|
|
def artifacts(self) -> dict[str, "mlflow.models.EvaluationArtifact"]:
|
|
"""
|
|
A dictionary mapping standardized artifact names (e.g. "roc_data") to
|
|
artifact content and location information
|
|
"""
|
|
return self._artifacts
|
|
|
|
@property
|
|
def run_id(self) -> str:
|
|
"""
|
|
The ID of the MLflow Run to which the evaluation results were logged.
|
|
"""
|
|
return self._run_id
|
|
|
|
@property
|
|
def tables(self) -> dict[str, "pd.DataFrame"]:
|
|
"""
|
|
A dictionary mapping standardized artifact names (e.g. "eval_results_table") to
|
|
corresponding table content as pandas DataFrame.
|
|
"""
|
|
eval_tables = {}
|
|
if self._run_id is None:
|
|
_logger.warning("Cannot load eval_results_table because run_id is not specified.")
|
|
return eval_tables
|
|
|
|
for table_name, table_path in self._artifacts.items():
|
|
path = urllib.parse.urlparse(table_path.uri).path
|
|
table_fileName = os.path.basename(path)
|
|
try:
|
|
eval_tables[table_name] = mlflow.load_table(table_fileName, run_ids=[self._run_id])
|
|
except Exception:
|
|
pass # Swallow the exception since we assume its not a table.
|
|
|
|
return eval_tables
|
|
|
|
|
|
@developer_stable
|
|
class ModelEvaluator(metaclass=ABCMeta):
|
|
@classmethod
|
|
@abstractmethod
|
|
def can_evaluate(cls, *, model_type, evaluator_config, **kwargs) -> bool:
|
|
"""
|
|
Args:
|
|
model_type: A string describing the model type (e.g., "regressor", "classifier", …).
|
|
evaluator_config: A dictionary of additional configurations for
|
|
the evaluator.
|
|
kwargs: For forwards compatibility, a placeholder for additional arguments
|
|
that may be added to the evaluation interface in the future.
|
|
|
|
Returns:
|
|
True if the evaluator can evaluate the specified model on the
|
|
specified dataset. False otherwise.
|
|
"""
|
|
|
|
@abstractmethod
|
|
def evaluate(
|
|
self,
|
|
*,
|
|
model_type,
|
|
dataset,
|
|
run_id,
|
|
evaluator_config,
|
|
model=None,
|
|
extra_metrics=None,
|
|
custom_artifacts=None,
|
|
predictions=None,
|
|
**kwargs,
|
|
):
|
|
"""
|
|
The abstract API to log metrics and artifacts, and return evaluation results.
|
|
|
|
Args:
|
|
model_type: A string describing the model type
|
|
(e.g., ``"regressor"``, ``"classifier"``, …).
|
|
dataset: An instance of `mlflow.models.evaluation.base._EvaluationDataset`
|
|
containing features and labels (optional) for model evaluation.
|
|
run_id: The ID of the MLflow Run to which to log results.
|
|
evaluator_config: A dictionary of additional configurations for
|
|
the evaluator.
|
|
model: A pyfunc model instance. If None, the model output is supposed to be found in
|
|
``dataset.predictions_data``.
|
|
extra_metrics: A list of :py:class:`EvaluationMetric` objects.
|
|
custom_artifacts: A list of callable custom artifact functions.
|
|
predictions: The column name of the model output column that is used for evaluation.
|
|
This is only used when a model returns a pandas dataframe that contains
|
|
multiple columns.
|
|
kwargs: For forwards compatibility, a placeholder for additional arguments that
|
|
may be added to the evaluation interface in the future.
|
|
|
|
Returns:
|
|
A :py:class:`mlflow.models.EvaluationResult` instance containing
|
|
evaluation metrics and artifacts for the model.
|
|
"""
|
|
|
|
|
|
def list_evaluators():
|
|
"""
|
|
Return a name list for all available Evaluators.
|
|
"""
|
|
# import _model_evaluation_registry inside function to avoid circuit importing
|
|
from mlflow.models.evaluation.evaluator_registry import _model_evaluation_registry
|
|
|
|
return list(_model_evaluation_registry._registry.keys())
|
|
|
|
|
|
@contextmanager
|
|
def _start_run_or_reuse_active_run():
|
|
"""
|
|
A manager context return:
|
|
- If there's an active run, return the active run.
|
|
- otherwise start a mflow run with the specified run_id,
|
|
if specified run_id is None, start a new run.
|
|
"""
|
|
active_run = mlflow.active_run()
|
|
if not active_run:
|
|
# Note `mlflow.start_run` throws if `run_id` is not found.
|
|
with mlflow.start_run() as run:
|
|
yield run
|
|
else:
|
|
yield active_run
|
|
|
|
|
|
# NB: We often pass around evaluator name, config, and its instance together. Ideally, the
|
|
# evaluator class should have name and config as class attributes, however, it was not
|
|
# designed that way. Adding them while keeping backward compatibility is not trivial.
|
|
# So, we use a dataclass to bundle them together.
|
|
@dataclass
|
|
class EvaluatorBundle:
|
|
name: str
|
|
evaluator: ModelEvaluator
|
|
config: dict[str, Any]
|
|
|
|
|
|
def _resolve_default_evaluator(model_type, default_config) -> list[EvaluatorBundle]:
|
|
"""
|
|
Determine which built-in evaluators should be used for the given model type by default.
|
|
|
|
Previously, MLflow evaluate API only had a single "default" evaluator used for all models like
|
|
classifier, regressor, etc. We split it into multiple built-in evaluators for different model
|
|
types for maintainability, but in order to maintain backward compatibility, we need to map
|
|
the "default" provided by users to the correct built-in evaluators.
|
|
|
|
Args:
|
|
model_type: A string describing the model type (e.g., "regressor", "classifier", …).
|
|
default_config: A dictionary of configurations for the "default" evaluator. If any
|
|
non-default built-in evaluator is applicable, this config will be applied to them.
|
|
"""
|
|
from mlflow.models.evaluation.evaluator_registry import _model_evaluation_registry
|
|
|
|
builtin_evaluators = []
|
|
for name in _model_evaluation_registry._registry:
|
|
evaluator = _model_evaluation_registry.get_evaluator(name)
|
|
if (
|
|
name != "default"
|
|
and _model_evaluation_registry.is_builtin(name)
|
|
and evaluator.can_evaluate(model_type=model_type, evaluator_config=default_config)
|
|
):
|
|
builtin_evaluators.append(EvaluatorBundle(name, evaluator, default_config))
|
|
|
|
# We should use DefaultEvaluator only if there is no other built-in evaluator applicable.
|
|
if not builtin_evaluators:
|
|
default_evaluator = _model_evaluation_registry.get_evaluator("default")
|
|
builtin_evaluators = [EvaluatorBundle("default", default_evaluator, default_config)]
|
|
|
|
return builtin_evaluators
|
|
|
|
|
|
def resolve_evaluators_and_configs(
|
|
evaluators: str | list[str] | None,
|
|
evaluator_config: dict[str, Any] | None,
|
|
model_type: str | None = None,
|
|
) -> list[EvaluatorBundle]:
|
|
"""
|
|
The `evaluators` and `evaluator_config` arguments of the `evaluate` API can be specified
|
|
in multiple ways. This function normalizes the arguments into a single format for easier
|
|
downstream processing.
|
|
|
|
Args:
|
|
evaluators: A string or a list of strings specifying the evaluators to use for model
|
|
evaluation. If None, all available evaluators will be used.
|
|
evaluator_config: A dictionary containing configuration items for the evaluators.
|
|
model_type: A string describing the model type (e.g., "regressor", "classifier", …).
|
|
|
|
Returns:
|
|
A list of EvaluatorBundle that contains name, evaluator, config for each evaluator.
|
|
"""
|
|
from mlflow.models.evaluation.evaluator_registry import _model_evaluation_registry as rg
|
|
|
|
# NB: The `databricks-agents` package must be installed to use the 'databricks-agent' model
|
|
# type. Ideally this check should be done in the 'databricks-agent' evaluator implementation,
|
|
# but we need to do it here because the code won't reach the evaluator implementation if the
|
|
# package is not installed.
|
|
if model_type == _ModelType.DATABRICKS_AGENT:
|
|
try:
|
|
import databricks.agents # noqa: F401
|
|
except ImportError as e:
|
|
raise MlflowException(
|
|
message="Databricks Agents SDK must be installed to use the "
|
|
f"`{_ModelType.DATABRICKS_AGENT}` model type. Run `pip install databricks-agents` "
|
|
"to install the package and try again.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
) from e
|
|
|
|
def check_nesting_config_dict(_evaluator_name_list, _evaluator_name_to_conf_map):
|
|
return isinstance(_evaluator_name_to_conf_map, dict) and all(
|
|
k in _evaluator_name_list and isinstance(v, dict)
|
|
for k, v in _evaluator_name_to_conf_map.items()
|
|
)
|
|
|
|
if evaluators is None:
|
|
# If no evaluators are specified, use all available evaluators.
|
|
evaluators = list(rg._registry.keys())
|
|
|
|
evaluator_config = evaluator_config or {}
|
|
if evaluator_config is not None and not any(
|
|
name in evaluator_config for name in evaluators
|
|
):
|
|
# If evaluator config is passed but any of available evaluator key is not
|
|
# in the evaluator config, we assume the evaluator config to be a flat dict,
|
|
# which is globally applied to all evaluators.
|
|
evaluator_config = dict.fromkeys(evaluators, evaluator_config)
|
|
|
|
# Filter out evaluators that cannot evaluate the model type.
|
|
resolved = []
|
|
for name in evaluators:
|
|
evaluator = rg.get_evaluator(name)
|
|
config = evaluator_config.get(name, {})
|
|
if evaluator.can_evaluate(model_type=model_type, evaluator_config=config):
|
|
resolved.append(EvaluatorBundle(name=name, evaluator=evaluator, config=config))
|
|
|
|
# If any of built-in evaluator can apply, skip "default" evaluator.
|
|
default = next((ev for ev in resolved if ev.name == "default"), None)
|
|
non_default_builtins = [
|
|
ev for ev in resolved if ev.name != "default" and rg.is_builtin(ev.name)
|
|
]
|
|
if default and non_default_builtins:
|
|
resolved.remove(default)
|
|
# Apply default config (passed like `evaluator_config={"default": config}`) to
|
|
# non-default built-in evaluators (e.g., ClassifierEvaluator) if they don't have
|
|
# explicitly specified configs. This is for backward compatibility where we only
|
|
# had a single "default" evaluator used for all models.
|
|
# For example, if the user passes this for a classifier model:
|
|
# evaluator_config = {"default": my_config}
|
|
# it should be equivalent to
|
|
# evaluator_config = {"classifier": my_config, "shap": my_config}
|
|
for ev in non_default_builtins:
|
|
ev.config = ev.config or default.config
|
|
|
|
return resolved
|
|
|
|
elif isinstance(evaluators, str):
|
|
# Single evaluator name specified
|
|
if not (evaluator_config is None or isinstance(evaluator_config, dict)):
|
|
raise MlflowException(
|
|
message="If `evaluators` argument is the name of an evaluator, evaluator_config"
|
|
" must be None or a dict containing config items for the evaluator.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
|
|
evaluator_config = evaluator_config or {}
|
|
if evaluators == "default":
|
|
# Previously we only had a single "default" evaluator used for all models.
|
|
# We need to map "default" to the new dedicated builtin evaluators.
|
|
return _resolve_default_evaluator(model_type, evaluator_config)
|
|
elif rg.is_registered(evaluators):
|
|
return [EvaluatorBundle(evaluators, rg.get_evaluator(evaluators), evaluator_config)]
|
|
else:
|
|
return []
|
|
|
|
elif isinstance(evaluators, list):
|
|
if evaluator_config is not None and not check_nesting_config_dict(
|
|
evaluators, evaluator_config
|
|
):
|
|
raise MlflowException(
|
|
message="If `evaluators` argument is an evaluator name list, evaluator_config "
|
|
"must be a dict containing mapping from evaluator name to individual "
|
|
"evaluator config dict.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
evaluator_config = evaluator_config or {}
|
|
|
|
# Previously we only had a single "default" evaluator used for all models.
|
|
# We need to map "default" to the new dedicated builtin evaluators.
|
|
resolved = []
|
|
for name in evaluators:
|
|
config = evaluator_config.get(name, {})
|
|
if name == "default":
|
|
builtin_evaluators = _resolve_default_evaluator(model_type, config)
|
|
resolved.extend(builtin_evaluators)
|
|
else:
|
|
resolved.append(EvaluatorBundle(name, rg.get_evaluator(name), config))
|
|
return resolved
|
|
else:
|
|
raise MlflowException(
|
|
message="Invalid `evaluators` and `evaluator_config` arguments. "
|
|
"Please refer to the documentation for correct usage.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
|
|
|
|
def _model_validation_contains_model_comparison(validation_thresholds):
|
|
"""
|
|
Helper function for determining if validation_thresholds contains
|
|
thresholds for model comparison: either min_relative_change or min_absolute_change
|
|
"""
|
|
if not validation_thresholds:
|
|
return False
|
|
thresholds = validation_thresholds.values()
|
|
return any(
|
|
threshold.min_relative_change or threshold.min_absolute_change for threshold in thresholds
|
|
)
|
|
|
|
|
|
_last_failed_evaluator = None
|
|
|
|
|
|
def _get_last_failed_evaluator():
|
|
"""
|
|
Return the evaluator name of the last failed evaluator when calling `evaluate`.
|
|
This can be used to check which evaluator fail when `evaluate` API fail.
|
|
"""
|
|
return _last_failed_evaluator
|
|
|
|
|
|
# DO NOT CHANGE THE ORDER OF THE ARGUMENTS
|
|
# The order of the arguments need to be preserved. You can add new arguments at the end
|
|
# of the argument list, but do not change the order of the existing arguments.
|
|
@record_usage_event(EvaluateEvent)
|
|
def _evaluate(
|
|
*,
|
|
model,
|
|
model_type,
|
|
model_id,
|
|
dataset,
|
|
run_id,
|
|
# The `evaluator_name_list` and `evaluator_name_to_conf_map` are not used by MLflow at all,
|
|
# but we need to keep these for backward compatibility.
|
|
evaluator_name_list,
|
|
evaluator_name_to_conf_map,
|
|
extra_metrics,
|
|
custom_artifacts,
|
|
predictions,
|
|
evaluators,
|
|
):
|
|
"""
|
|
The public API "evaluate" will verify argument first, and then pass normalized arguments
|
|
to the _evaluate method.
|
|
"""
|
|
global _last_failed_evaluator
|
|
_last_failed_evaluator = None
|
|
|
|
client = MlflowClient()
|
|
|
|
model_uuid = getattr(model, "metadata", None)
|
|
|
|
if model_uuid is not None:
|
|
model_uuid = model_uuid.model_uuid
|
|
dataset._log_dataset_tag(client, run_id, model_uuid)
|
|
|
|
eval_results = []
|
|
should_enable_tracing = model is not None # Do not enable tracing if static dataset is provided
|
|
for eval_ in evaluators:
|
|
_logger.debug(f"Evaluating the model with the {eval_.name} evaluator.")
|
|
_last_failed_evaluator = eval_.name
|
|
if eval_.evaluator.can_evaluate(model_type=model_type, evaluator_config=eval_.config):
|
|
with configure_autologging_for_evaluation(enable_tracing=should_enable_tracing):
|
|
eval_result = eval_.evaluator.evaluate(
|
|
model=model,
|
|
model_type=model_type,
|
|
model_id=model_id,
|
|
dataset=dataset,
|
|
run_id=run_id,
|
|
evaluator_config=eval_.config,
|
|
extra_metrics=extra_metrics,
|
|
custom_artifacts=custom_artifacts,
|
|
predictions=predictions,
|
|
)
|
|
|
|
if eval_result is not None:
|
|
eval_results.append(eval_result)
|
|
|
|
_last_failed_evaluator = None
|
|
|
|
if len(eval_results) == 0:
|
|
raise MlflowException(
|
|
message="The model could not be evaluated by any of the registered evaluators, please "
|
|
"verify that the model type and other configs are set correctly.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
|
|
merged_eval_result = EvaluationResult({}, {}, None)
|
|
|
|
for eval_result in eval_results:
|
|
merged_eval_result.metrics.update(eval_result.metrics)
|
|
merged_eval_result.artifacts.update(eval_result.artifacts)
|
|
|
|
return merged_eval_result
|
|
|
|
|
|
def _get_model_from_function(fn):
|
|
from mlflow.pyfunc.model import _PythonModelPyfuncWrapper
|
|
|
|
class ModelFromFunction(mlflow.pyfunc.PythonModel):
|
|
def predict(self, context, model_input: pd.DataFrame):
|
|
return fn(model_input)
|
|
|
|
python_model = ModelFromFunction()
|
|
return _PythonModelPyfuncWrapper(python_model, None, None)
|
|
|
|
|
|
def _is_model_deployment_endpoint_uri(model: Any) -> bool:
|
|
if not isinstance(model, str):
|
|
return False
|
|
|
|
from mlflow.metrics.genai.model_utils import _parse_model_uri
|
|
|
|
try:
|
|
schema, path = _parse_model_uri(model)
|
|
return schema in ["endpoints", "apps"]
|
|
except MlflowException:
|
|
return False
|
|
|
|
|
|
def _get_model_from_deployment_endpoint_uri(
|
|
endpoint_uri: str, params: dict[str, Any] | None = None
|
|
):
|
|
from mlflow.metrics.genai.model_utils import _parse_model_uri
|
|
from mlflow.pyfunc.model import ModelFromDeploymentEndpoint, _PythonModelPyfuncWrapper
|
|
|
|
_, endpoint = _parse_model_uri(endpoint_uri)
|
|
params = params or {}
|
|
|
|
python_model = ModelFromDeploymentEndpoint(endpoint, params)
|
|
return _PythonModelPyfuncWrapper(python_model, None, None)
|
|
|
|
|
|
def evaluate(
|
|
model=None,
|
|
data=None,
|
|
*,
|
|
model_type=None,
|
|
targets=None,
|
|
predictions=None,
|
|
dataset_path=None,
|
|
feature_names=None,
|
|
evaluators=None,
|
|
evaluator_config=None,
|
|
extra_metrics=None,
|
|
custom_artifacts=None,
|
|
env_manager="local",
|
|
model_config=None,
|
|
inference_params=None,
|
|
model_id=None,
|
|
_called_from_genai_evaluate=False,
|
|
):
|
|
'''
|
|
Evaluate the model performance on given data and selected metrics.
|
|
|
|
This function evaluates a PyFunc model or custom callable on the specified dataset using
|
|
specified ``evaluators``, and logs resulting metrics & artifacts to MLflow tracking server.
|
|
Users can also skip setting ``model`` and put the model outputs in ``data`` directly for
|
|
evaluation. For detailed information, please read
|
|
`the Model Evaluation documentation <../../model-evaluation/index.html>`_.
|
|
|
|
Default Evaluator behavior:
|
|
- The default evaluator, which can be invoked with ``evaluators="default"`` or
|
|
``evaluators=None``, supports model types listed below. For each pre-defined model type, the
|
|
default evaluator evaluates your model on a selected set of metrics and generate artifacts
|
|
like plots. Please find more details below.
|
|
|
|
- For both the ``"regressor"`` and ``"classifier"`` model types, the default evaluator
|
|
generates model summary plots and feature importance plots using
|
|
`SHAP <https://shap.readthedocs.io/en/latest/index.html>`_.
|
|
|
|
- For regressor models, the default evaluator additionally logs:
|
|
- **metrics**: example_count, mean_absolute_error, mean_squared_error,
|
|
root_mean_squared_error, sum_on_target, mean_on_target, r2_score, max_error,
|
|
mean_absolute_percentage_error.
|
|
|
|
- For binary classifiers, the default evaluator additionally logs:
|
|
- **metrics**: true_negatives, false_positives, false_negatives, true_positives, recall,
|
|
precision, f1_score, accuracy_score, example_count, log_loss, roc_auc,
|
|
precision_recall_auc.
|
|
- **artifacts**: lift curve plot, precision-recall plot, ROC plot.
|
|
|
|
- For multiclass classifiers, the default evaluator additionally logs:
|
|
- **metrics**: accuracy_score, example_count, f1_score_micro, f1_score_macro, log_loss
|
|
- **artifacts**: A CSV file for "per_class_metrics" (per-class metrics includes
|
|
true_negatives/false_positives/false_negatives/true_positives/recall/precision/roc_auc,
|
|
precision_recall_auc), precision-recall merged curves plot, ROC merged curves plot.
|
|
|
|
- For question-answering models, the default evaluator logs:
|
|
- **metrics**: ``exact_match``, ``token_count``, `toxicity`_ (requires `evaluate`_,
|
|
`torch`_, `flesch_kincaid_grade_level`_ (requires `textstat`_) and `ari_grade_level`_.
|
|
- **artifacts**: A JSON file containing the inputs, outputs, targets (if the ``targets``
|
|
argument is supplied), and per-row metrics of the model in tabular format.
|
|
|
|
.. _toxicity:
|
|
https://huggingface.co/spaces/evaluate-measurement/toxicity
|
|
|
|
.. _torch:
|
|
https://pytorch.org/get-started/locally/
|
|
|
|
.. _transformers:
|
|
https://huggingface.co/docs/transformers/installation
|
|
|
|
.. _ari_grade_level:
|
|
https://en.wikipedia.org/wiki/Automated_readability_index
|
|
|
|
.. _flesch_kincaid_grade_level:
|
|
https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests#Flesch%E2%80%93Kincaid_grade_level
|
|
|
|
.. _evaluate:
|
|
https://pypi.org/project/evaluate
|
|
|
|
.. _textstat:
|
|
https://pypi.org/project/textstat
|
|
|
|
- For text-summarization models, the default evaluator logs:
|
|
- **metrics**: ``token_count``, `ROUGE`_ (requires `evaluate`_, `nltk`_, and
|
|
`rouge_score`_ to be installed), `toxicity`_ (requires `evaluate`_, `torch`_,
|
|
`transformers`_), `ari_grade_level`_ (requires `textstat`_),
|
|
`flesch_kincaid_grade_level`_ (requires `textstat`_).
|
|
- **artifacts**: A JSON file containing the inputs, outputs, targets (if the ``targets``
|
|
argument is supplied), and per-row metrics of the model in the tabular format.
|
|
|
|
.. _ROUGE:
|
|
https://huggingface.co/spaces/evaluate-metric/rouge
|
|
|
|
.. _toxicity:
|
|
https://huggingface.co/spaces/evaluate-measurement/toxicity
|
|
|
|
.. _torch:
|
|
https://pytorch.org/get-started/locally/
|
|
|
|
.. _transformers:
|
|
https://huggingface.co/docs/transformers/installation
|
|
|
|
.. _ari_grade_level:
|
|
https://en.wikipedia.org/wiki/Automated_readability_index
|
|
|
|
.. _flesch_kincaid_grade_level:
|
|
https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests#Flesch%E2%80%93Kincaid_grade_level
|
|
|
|
.. _evaluate:
|
|
https://pypi.org/project/evaluate
|
|
|
|
.. _nltk:
|
|
https://pypi.org/project/nltk
|
|
|
|
.. _rouge_score:
|
|
https://pypi.org/project/rouge-score
|
|
|
|
.. _textstat:
|
|
https://pypi.org/project/textstat
|
|
|
|
- For text models, the default evaluator logs:
|
|
- **metrics**: ``token_count``, `toxicity`_ (requires `evaluate`_, `torch`_,
|
|
`transformers`_), `ari_grade_level`_ (requires `textstat`_),
|
|
`flesch_kincaid_grade_level`_ (requires `textstat`_).
|
|
- **artifacts**: A JSON file containing the inputs, outputs, targets (if the ``targets``
|
|
argument is supplied), and per-row metrics of the model in tabular format.
|
|
|
|
.. _evaluate:
|
|
https://pypi.org/project/evaluate
|
|
|
|
.. _toxicity:
|
|
https://huggingface.co/spaces/evaluate-measurement/toxicity
|
|
|
|
.. _torch:
|
|
https://pytorch.org/get-started/locally/
|
|
|
|
.. _transformers:
|
|
https://huggingface.co/docs/transformers/installation
|
|
|
|
.. _ari_grade_level:
|
|
https://en.wikipedia.org/wiki/Automated_readability_index
|
|
|
|
.. _flesch_kincaid_grade_level:
|
|
https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests#Flesch%E2%80%93Kincaid_grade_level
|
|
|
|
.. _textstat:
|
|
https://pypi.org/project/textstat
|
|
|
|
- For retriever models, the default evaluator logs:
|
|
- **metrics**: :mod:`precision_at_k(k) <mlflow.metrics.precision_at_k>`,
|
|
:mod:`recall_at_k(k) <mlflow.metrics.recall_at_k>` and
|
|
:mod:`ndcg_at_k(k) <mlflow.metrics.ndcg_at_k>` - all have a default value of
|
|
``retriever_k`` = 3.
|
|
- **artifacts**: A JSON file containing the inputs, outputs, targets, and per-row metrics
|
|
of the model in tabular format.
|
|
|
|
- For sklearn models, the default evaluator additionally logs the model's evaluation criterion
|
|
(e.g. mean accuracy for a classifier) computed by `model.score` method.
|
|
|
|
- The metrics/artifacts listed above are logged to the active MLflow run.
|
|
If no active run exists, a new MLflow run is created for logging these metrics and
|
|
artifacts.
|
|
|
|
- Additionally, information about the specified dataset - hash, name (if specified), path
|
|
(if specified), and the UUID of the model that evaluated it - is logged to the
|
|
``mlflow.datasets`` tag.
|
|
|
|
- The available ``evaluator_config`` options for the default evaluator include:
|
|
- **log_model_explainability**: A boolean value specifying whether or not to log model
|
|
explainability insights, default value is True.
|
|
- **log_explainer**: If True, log the explainer used to compute model explainability
|
|
insights as a model. Default value is False.
|
|
- **explainability_algorithm**: A string to specify the SHAP Explainer algorithm for model
|
|
explainability. Supported algorithm includes: 'exact', 'permutation', 'partition',
|
|
'kernel'.
|
|
If not set, ``shap.Explainer`` is used with the "auto" algorithm, which chooses the best
|
|
Explainer based on the model.
|
|
- **explainability_nsamples**: The number of sample rows to use for computing model
|
|
explainability insights. Default value is 2000.
|
|
- **explainability_kernel_link**: The kernel link function used by shap kernel explainer.
|
|
Available values are "identity" and "logit". Default value is "identity".
|
|
- **max_classes_for_multiclass_roc_pr**:
|
|
For multiclass classification tasks, the maximum number of classes for which to log
|
|
the per-class ROC curve and Precision-Recall curve. If the number of classes is
|
|
larger than the configured maximum, these curves are not logged.
|
|
- **metric_prefix**: An optional prefix to prepend to the name of each metric and artifact
|
|
produced during evaluation.
|
|
- **log_metrics_with_dataset_info**: A boolean value specifying whether or not to include
|
|
information about the evaluation dataset in the name of each metric logged to MLflow
|
|
Tracking during evaluation, default value is True.
|
|
- **pos_label**: If specified, the positive label to use when computing classification
|
|
metrics such as precision, recall, f1, etc. for binary classification models. For
|
|
multiclass classification and regression models, this parameter will be ignored.
|
|
- **average**: The averaging method to use when computing classification metrics such as
|
|
precision, recall, f1, etc. for multiclass classification models
|
|
(default: ``'weighted'``). For binary classification and regression models, this
|
|
parameter will be ignored.
|
|
- **sample_weights**: Weights for each sample to apply when computing model performance
|
|
metrics.
|
|
- **col_mapping**: A dictionary mapping column names in the input dataset or output
|
|
predictions to column names used when invoking the evaluation functions.
|
|
- **retriever_k**: A parameter used when ``model_type="retriever"`` as the number of
|
|
top-ranked retrieved documents to use when computing the built-in metric
|
|
:mod:`precision_at_k(k) <mlflow.metrics.precision_at_k>`,
|
|
:mod:`recall_at_k(k) <mlflow.metrics.recall_at_k>` and
|
|
:mod:`ndcg_at_k(k) <mlflow.metrics.ndcg_at_k>`. Default value is 3. For all other
|
|
model types, this parameter will be ignored.
|
|
|
|
- Limitations of evaluation dataset:
|
|
- For classification tasks, dataset labels are used to infer the total number of classes.
|
|
- For binary classification tasks, the negative label value must be 0 or -1 or False, and
|
|
the positive label value must be 1 or True.
|
|
|
|
- Limitations of metrics/artifacts computation:
|
|
- For classification tasks, some metric and artifact computations require the model to
|
|
output class probabilities. Currently, for scikit-learn models, the default evaluator
|
|
calls the ``predict_proba`` method on the underlying model to obtain probabilities. For
|
|
other model types, the default evaluator does not compute metrics/artifacts that require
|
|
probability outputs.
|
|
|
|
- Limitations of default evaluator logging model explainability insights:
|
|
- The ``shap.Explainer`` ``auto`` algorithm uses the ``Linear`` explainer for linear models
|
|
and the ``Tree`` explainer for tree models. Because SHAP's ``Linear`` and ``Tree``
|
|
explainers do not support multi-class classification, the default evaluator falls back to
|
|
using the ``Exact`` or ``Permutation`` explainers for multi-class classification tasks.
|
|
- Logging model explainability insights is not currently supported for PySpark models.
|
|
- The evaluation dataset label values must be numeric or boolean, all feature values
|
|
must be numeric, and each feature column must only contain scalar values.
|
|
|
|
- Limitations when environment restoration is enabled:
|
|
- When environment restoration is enabled for the evaluated model (i.e. a non-local
|
|
``env_manager`` is specified), the model is loaded as a client that invokes a MLflow
|
|
Model Scoring Server process in an independent Python environment with the model's
|
|
training time dependencies installed. As such, methods like ``predict_proba`` (for
|
|
probability outputs) or ``score`` (computes the evaluation criterian for sklearn models)
|
|
of the model become inaccessible and the default evaluator does not compute metrics or
|
|
artifacts that require those methods.
|
|
- Because the model is an MLflow Model Server process, SHAP explanations are slower to
|
|
compute. As such, model explainaibility is disabled when a non-local ``env_manager``
|
|
specified, unless the ``evaluator_config`` option **log_model_explainability** is
|
|
explicitly set to ``True``.
|
|
|
|
Args:
|
|
model: Optional. If specified, it should be one of the following:
|
|
|
|
- A pyfunc model instance
|
|
- A URI referring to a pyfunc model
|
|
- A URI referring to an MLflow Deployments endpoint e.g. ``"endpoints:/my-chat"``
|
|
- A callable function: This function should be able to take in model input and
|
|
return predictions. It should follow the signature of the
|
|
:py:func:`predict <mlflow.pyfunc.PyFuncModel.predict>` method. Here's an example
|
|
of a valid function:
|
|
|
|
.. code-block:: python
|
|
|
|
model = mlflow.pyfunc.load_model(model_uri)
|
|
|
|
|
|
def fn(model_input):
|
|
return model.predict(model_input)
|
|
|
|
If omitted, it indicates a static dataset will be used for evaluation instead of a
|
|
model. In this case, the ``data`` argument must be a Pandas DataFrame or an mlflow
|
|
PandasDataset that contains model outputs, and the ``predictions`` argument must be the
|
|
name of the column in ``data`` that contains model outputs.
|
|
|
|
data: One of the
|
|
following:
|
|
|
|
- A numpy array or list of evaluation features, excluding labels.
|
|
- A Pandas DataFrame containing evaluation features, labels, and optionally model
|
|
outputs. Model outputs are required to be provided when model is unspecified.
|
|
If ``feature_names`` argument not specified, all columns except for the label
|
|
column and predictions column are regarded as feature columns. Otherwise,
|
|
only column names present in ``feature_names`` are regarded as feature columns.
|
|
- A Spark DataFrame containing evaluation features and labels. If
|
|
``feature_names`` argument not specified, all columns except for the label
|
|
column are regarded as feature columns. Otherwise, only column names present in
|
|
``feature_names`` are regarded as feature columns. Only the first 10000 rows in
|
|
the Spark DataFrame will be used as evaluation data.
|
|
- A :py:class:`mlflow.data.dataset.Dataset` instance containing evaluation
|
|
features, labels, and optionally model outputs. Model outputs are only supported
|
|
with a PandasDataset. Model outputs are required when model is unspecified, and
|
|
should be specified via the ``predictions`` property of the PandasDataset.
|
|
|
|
model_type: (Optional) A string describing the model type. The default evaluator
|
|
supports the following model types:
|
|
|
|
- ``'classifier'``
|
|
- ``'regressor'``
|
|
- ``'question-answering'``
|
|
- ``'text-summarization'``
|
|
- ``'text'``
|
|
- ``'retriever'``
|
|
|
|
If no ``model_type`` is specified, then you must provide a a list of
|
|
metrics to compute via the ``extra_metrics`` param.
|
|
|
|
.. note::
|
|
``'question-answering'``, ``'text-summarization'``, ``'text'``, and
|
|
``'retriever'`` are experimental and may be changed or removed in a
|
|
future release.
|
|
|
|
targets: If ``data`` is a numpy array or list, a numpy array or list of evaluation
|
|
labels. If ``data`` is a DataFrame, the string name of a column from ``data``
|
|
that contains evaluation labels. Required for classifier and regressor models,
|
|
but optional for question-answering, text-summarization, and text models. If
|
|
``data`` is a :py:class:`mlflow.data.dataset.Dataset` that defines targets,
|
|
then ``targets`` is optional.
|
|
|
|
predictions: Optional. The name of the column that contains model outputs.
|
|
|
|
- When ``model`` is specified and outputs multiple columns, ``predictions`` can be used
|
|
to specify the name of the column that will be used to store model outputs for
|
|
evaluation.
|
|
- When ``model`` is not specified and ``data`` is a pandas dataframe,
|
|
``predictions`` can be used to specify the name of the column in ``data`` that
|
|
contains model outputs.
|
|
|
|
.. code-block:: python
|
|
:caption: Example usage of predictions
|
|
|
|
# Evaluate a model that outputs multiple columns
|
|
data = pd.DataFrame({"question": ["foo"]})
|
|
|
|
|
|
def model(inputs):
|
|
return pd.DataFrame({"answer": ["bar"], "source": ["baz"]})
|
|
|
|
|
|
results = evaluate(
|
|
model=model,
|
|
data=data,
|
|
predictions="answer",
|
|
# other arguments if needed
|
|
)
|
|
|
|
# Evaluate a static dataset
|
|
data = pd.DataFrame({"question": ["foo"], "answer": ["bar"], "source": ["baz"]})
|
|
results = evaluate(
|
|
data=data,
|
|
predictions="answer",
|
|
# other arguments if needed
|
|
)
|
|
dataset_path: (Optional) The path where the data is stored. Must not contain double
|
|
quotes (``"``). If specified, the path is logged to the ``mlflow.datasets``
|
|
tag for lineage tracking purposes.
|
|
|
|
feature_names: (Optional) A list. If the ``data`` argument is a numpy array or list,
|
|
``feature_names`` is a list of the feature names for each feature. If
|
|
``feature_names=None``, then the ``feature_names`` are generated using the
|
|
format ``feature_{feature_index}``. If the ``data`` argument is a Pandas
|
|
DataFrame or a Spark DataFrame, ``feature_names`` is a list of the names
|
|
of the feature columns in the DataFrame. If ``feature_names=None``, then
|
|
all columns except the label column and the predictions column are
|
|
regarded as feature columns.
|
|
|
|
evaluators: The name of the evaluator to use for model evaluation, or a list of
|
|
evaluator names. If unspecified, all evaluators capable of evaluating the
|
|
specified model on the specified dataset are used. The default evaluator
|
|
can be referred to by the name ``"default"``. To see all available
|
|
evaluators, call :py:func:`mlflow.models.list_evaluators`.
|
|
|
|
evaluator_config: A dictionary of additional configurations to supply to the evaluator.
|
|
If multiple evaluators are specified, each configuration should be
|
|
supplied as a nested dictionary whose key is the evaluator name.
|
|
|
|
extra_metrics:
|
|
(Optional) A list of :py:class:`EvaluationMetric <mlflow.models.EvaluationMetric>`
|
|
objects. These metrics are computed in addition to the default metrics associated with
|
|
pre-defined `model_type`, and setting `model_type=None` will only compute the metrics
|
|
specified in `extra_metrics`. See the `mlflow.metrics` module for more information about
|
|
the builtin metrics and how to define extra metrics.
|
|
|
|
.. code-block:: python
|
|
:caption: Example usage of extra metrics
|
|
|
|
import mlflow
|
|
import numpy as np
|
|
|
|
|
|
def root_mean_squared_error(eval_df, _builtin_metrics):
|
|
return np.sqrt((np.abs(eval_df["prediction"] - eval_df["target"]) ** 2).mean())
|
|
|
|
|
|
rmse_metric = mlflow.models.make_metric(
|
|
eval_fn=root_mean_squared_error,
|
|
greater_is_better=False,
|
|
)
|
|
mlflow.evaluate(..., extra_metrics=[rmse_metric])
|
|
|
|
custom_artifacts:
|
|
(Optional) A list of custom artifact functions with the following signature:
|
|
|
|
.. code-block:: python
|
|
|
|
def custom_artifact(
|
|
eval_df: Union[pandas.Dataframe, pyspark.sql.DataFrame],
|
|
builtin_metrics: Dict[str, float],
|
|
artifacts_dir: str,
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Args:
|
|
eval_df:
|
|
A Pandas or Spark DataFrame containing ``prediction`` and ``target``
|
|
column. The ``prediction`` column contains the predictions made by the
|
|
model. The ``target`` column contains the corresponding labels to the
|
|
predictions made on that row.
|
|
builtin_metrics:
|
|
A dictionary containing the metrics calculated by the default evaluator.
|
|
The keys are the names of the metrics and the values are the scalar
|
|
values of the metrics. Refer to the DefaultEvaluator behavior section
|
|
for what metrics will be returned based on the type of model (i.e.
|
|
classifier or regressor).
|
|
artifacts_dir:
|
|
A temporary directory path that can be used by the custom artifacts
|
|
function to temporarily store produced artifacts. The directory will be
|
|
deleted after the artifacts are logged.
|
|
|
|
Returns:
|
|
A dictionary that maps artifact names to artifact objects
|
|
(e.g. a Matplotlib Figure) or to artifact paths within ``artifacts_dir``.
|
|
"""
|
|
...
|
|
|
|
Object types that artifacts can be represented as:
|
|
|
|
- A string uri representing the file path to the artifact. MLflow will infer the
|
|
type of the artifact based on the file extension.
|
|
- A string representation of a JSON object. This will be saved as a .json artifact.
|
|
- Pandas DataFrame. This will be resolved as a CSV artifact.
|
|
- Numpy array. This will be saved as a .npy artifact.
|
|
- Matplotlib Figure. This will be saved as an image artifact. Note that
|
|
``matplotlib.pyplot.savefig`` is called behind the scene with default
|
|
configurations.
|
|
To customize, either save the figure with the desired configurations and return
|
|
its file path or define customizations through environment variables in
|
|
``matplotlib.rcParams``.
|
|
- Other objects will be attempted to be pickled with the default protocol.
|
|
|
|
.. code-block:: python
|
|
:caption: Example usage of custom artifacts
|
|
|
|
import mlflow
|
|
import matplotlib.pyplot as plt
|
|
|
|
|
|
def scatter_plot(eval_df, builtin_metrics, artifacts_dir):
|
|
plt.scatter(eval_df["prediction"], eval_df["target"])
|
|
plt.xlabel("Targets")
|
|
plt.ylabel("Predictions")
|
|
plt.title("Targets vs. Predictions")
|
|
plt.savefig(os.path.join(artifacts_dir, "example.png"))
|
|
plt.close()
|
|
return {"pred_target_scatter": os.path.join(artifacts_dir, "example.png")}
|
|
|
|
|
|
def pred_sample(eval_df, _builtin_metrics, _artifacts_dir):
|
|
return {"pred_sample": pred_sample.head(10)}
|
|
|
|
|
|
mlflow.evaluate(..., custom_artifacts=[scatter_plot, pred_sample])
|
|
|
|
env_manager: Specify an environment manager to load the candidate ``model`` in
|
|
isolated Python environments and restore their
|
|
dependencies. Default value is ``local``, and the following values are
|
|
supported:
|
|
|
|
- ``virtualenv``: (Recommended) Use virtualenv to restore the python
|
|
environment that was used to train the model.
|
|
- ``conda``: Use Conda to restore the software environment that was used
|
|
to train the model.
|
|
- ``local``: Use the current Python environment for model inference, which
|
|
may differ from the environment used to train the model and may lead to
|
|
errors or invalid predictions.
|
|
|
|
model_config: the model configuration to use for loading the model with pyfunc. Inspect
|
|
the model's pyfunc flavor to know which keys are supported for your
|
|
specific model. If not indicated, the default model configuration
|
|
from the model is used (if any).
|
|
|
|
inference_params: (Optional) A dictionary of inference parameters to be passed to the model
|
|
when making predictions, such as ``{"max_tokens": 100}``. This is only used when
|
|
the ``model`` is an MLflow Deployments endpoint URI e.g. ``"endpoints:/my-chat"``
|
|
|
|
model_id: (Optional) The ID of the MLflow LoggedModel or Model Version to which the
|
|
evaluation results (e.g. metrics and traces) will be linked. If `model_id` is not
|
|
specified but `model` is specified, the ID from `model` will be used.
|
|
|
|
_called_from_genai_evaluate: (Optional) Only used internally.
|
|
|
|
Returns:
|
|
An :py:class:`mlflow.models.EvaluationResult` instance containing
|
|
metrics of evaluating the model with the given dataset.
|
|
'''
|
|
from mlflow.models.evaluation.evaluator_registry import _model_evaluation_registry
|
|
from mlflow.pyfunc import PyFuncModel, _load_model_or_server, _ServedPyFuncModel
|
|
from mlflow.utils import env_manager as _EnvManager
|
|
|
|
# Inference params are currently only supported for passing a deployment endpoint as the model.
|
|
# TODO: We should support inference_params for other model types
|
|
|
|
if inference_params is not None and not _is_model_deployment_endpoint_uri(model):
|
|
raise MlflowException(
|
|
message="The inference_params argument can only be specified when the model "
|
|
"is an MLflow Deployments endpoint URI like `endpoints:/my-chat`",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
|
|
if evaluator_config is not None:
|
|
col_mapping = evaluator_config.get("col_mapping", {})
|
|
|
|
if isinstance(targets, str):
|
|
targets = col_mapping.get(targets, targets)
|
|
|
|
if isinstance(predictions, str):
|
|
predictions = col_mapping.get(predictions, predictions)
|
|
|
|
if data is None:
|
|
raise MlflowException(
|
|
message="The data argument cannot be None.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
|
|
_EnvManager.validate(env_manager)
|
|
|
|
# If Dataset is provided, the targets can only be specified by the Dataset,
|
|
# not the targets parameters of the mlflow.evaluate() API.
|
|
if isinstance(data, Dataset) and targets is not None:
|
|
raise MlflowException(
|
|
message="The top-level targets parameter should not be specified since a Dataset "
|
|
"is used. Please only specify the targets column name in the Dataset. For example: "
|
|
"`data = mlflow.data.from_pandas(df=X.assign(y=y), targets='y')`. "
|
|
"Meanwhile, please specify `mlflow.evaluate(..., targets=None, ...)`.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
# If Dataset is provided and model is None, then the predictions can only be specified by the
|
|
# Dataset, not the predictions parameters of the mlflow.evaluate() API.
|
|
if isinstance(data, Dataset) and model is None and predictions is not None:
|
|
raise MlflowException(
|
|
message="The top-level predictions parameter should not be specified since a Dataset "
|
|
"is used. Please only specify the predictions column name in the Dataset. For example:"
|
|
" `data = mlflow.data.from_pandas(df=X.assign(y=y), predictions='y')`"
|
|
"Meanwhile, please specify `mlflow.evaluate(..., predictions=None, ...)`.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
# If Dataset is provided and model is specified, then the data.predictions cannot be specified.
|
|
if (
|
|
isinstance(data, Dataset)
|
|
and model is not None
|
|
and getattr(data, "predictions", None) is not None
|
|
):
|
|
raise MlflowException(
|
|
message="The predictions parameter should not be specified in the Dataset since a "
|
|
"model is specified. Please remove the predictions column from the Dataset.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
|
|
if model_type in [_ModelType.REGRESSOR, _ModelType.CLASSIFIER]:
|
|
if isinstance(data, Dataset):
|
|
if getattr(data, "targets", None) is not None:
|
|
targets = data.targets
|
|
else:
|
|
raise MlflowException(
|
|
message="The targets column name must be specified in the provided Dataset "
|
|
f"for {model_type} models. For example: "
|
|
"`data = mlflow.data.from_pandas(df=X.assign(y=y), targets='y')`",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
else:
|
|
if targets is None:
|
|
raise MlflowException(
|
|
f"The targets argument must be specified for {model_type} models.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
elif model_type is None:
|
|
if not extra_metrics:
|
|
raise MlflowException(
|
|
message="The extra_metrics argument must be specified model_type is None.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
|
|
specified_model_id = model_id
|
|
model_id = None
|
|
if isinstance(model, str):
|
|
model_id = _parse_model_id_if_present(model)
|
|
if _is_model_deployment_endpoint_uri(model):
|
|
model = _get_model_from_deployment_endpoint_uri(model, inference_params)
|
|
else:
|
|
model = _load_model_or_server(model, env_manager, model_config)
|
|
elif env_manager != _EnvManager.LOCAL:
|
|
raise MlflowException(
|
|
message="The model argument must be a string URI referring to an MLflow model when a "
|
|
"non-local env_manager is specified.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
elif isinstance(model, PyFuncModel):
|
|
model_id = model.model_id
|
|
if model_config:
|
|
raise MlflowException(
|
|
message="Indicating ``model_config`` when passing a `PyFuncModel`` object as "
|
|
"model argument is not allowed. If you need to change the model configuration "
|
|
"for the evaluation model, use "
|
|
"``mlflow.pyfunc.load_model(model_uri, model_config=<value>)`` and indicate "
|
|
"the desired configuration there.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
elif callable(model):
|
|
model = _get_model_from_function(model)
|
|
elif model is not None:
|
|
raise MlflowException(
|
|
message="The model argument must be a string URI referring to an MLflow model, "
|
|
"an MLflow Deployments endpoint URI, an instance of `mlflow.pyfunc.PyFuncModel`, "
|
|
"a function, or None.",
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
|
|
# If model_id is specified, verify it matches the derived model_id
|
|
if specified_model_id is not None and model_id is not None and specified_model_id != model_id:
|
|
raise MlflowException(
|
|
message=(
|
|
f"The specified value of the 'model_id' parameter '{specified_model_id}' "
|
|
f"contradicts the model_id '{model_id}' associated with the model. Please ensure "
|
|
f"they match or omit the 'model_id' parameter."
|
|
),
|
|
error_code=INVALID_PARAMETER_VALUE,
|
|
)
|
|
|
|
# Use specified model_id if provided, otherwise use derived model_id
|
|
model_id = specified_model_id if specified_model_id is not None else model_id
|
|
# If none of the model_id and model is specified, use the active model_id
|
|
model_id = model_id or mlflow.get_active_model_id()
|
|
|
|
evaluators: list[EvaluatorBundle] = resolve_evaluators_and_configs(
|
|
evaluators, evaluator_config, model_type
|
|
)
|
|
|
|
# NB: MLflow do not use either of these two variables. However, we need to pass these to
|
|
# _evaluate() function for backward compatibility.
|
|
evaluator_name_list = [evaluator.name for evaluator in evaluators]
|
|
evaluator_name_to_conf_map = {evaluator.name: evaluator.config for evaluator in evaluators}
|
|
|
|
with _start_run_or_reuse_active_run() as run:
|
|
if not isinstance(data, Dataset):
|
|
# Convert data to `mlflow.data.dataset.Dataset`.
|
|
if model is None:
|
|
data = convert_data_to_mlflow_dataset(
|
|
data=data, targets=targets, predictions=predictions
|
|
)
|
|
else:
|
|
data = convert_data_to_mlflow_dataset(data=data, targets=targets)
|
|
|
|
from mlflow.data.pyfunc_dataset_mixin import PyFuncConvertibleDatasetMixin
|
|
|
|
# model_id could be None
|
|
with _set_active_model(model_id=model_id) if model_id else nullcontext():
|
|
if isinstance(data, Dataset) and issubclass(
|
|
data.__class__, PyFuncConvertibleDatasetMixin
|
|
):
|
|
dataset = data.to_evaluation_dataset(dataset_path, feature_names)
|
|
|
|
# Use metric_prefix configured for builtin evaluators as a dataset tag
|
|
context = None
|
|
for e in evaluators:
|
|
if _model_evaluation_registry.is_builtin(e.name) and e.config.get(
|
|
"metric_prefix"
|
|
):
|
|
context = e.config.get("metric_prefix")
|
|
break
|
|
|
|
client = MlflowClient()
|
|
tags = [InputTag(key=MLFLOW_DATASET_CONTEXT, value=context)] if context else []
|
|
dataset_input = DatasetInput(dataset=data._to_mlflow_entity(), tags=tags)
|
|
client.log_inputs(
|
|
run.info.run_id,
|
|
[dataset_input],
|
|
models=[LoggedModelInput(model_id)] if model_id else None,
|
|
)
|
|
else:
|
|
dataset = EvaluationDataset(
|
|
data,
|
|
targets=targets,
|
|
path=dataset_path,
|
|
feature_names=feature_names,
|
|
predictions=predictions,
|
|
)
|
|
predictions_expected_in_model_output = predictions if model is not None else None
|
|
|
|
try:
|
|
evaluate_result = _evaluate(
|
|
model=model,
|
|
model_type=model_type,
|
|
model_id=model_id,
|
|
dataset=dataset,
|
|
run_id=run.info.run_id,
|
|
evaluator_name_list=evaluator_name_list,
|
|
evaluator_name_to_conf_map=evaluator_name_to_conf_map,
|
|
extra_metrics=extra_metrics,
|
|
custom_artifacts=custom_artifacts,
|
|
predictions=predictions_expected_in_model_output,
|
|
evaluators=evaluators,
|
|
)
|
|
finally:
|
|
if isinstance(model, _ServedPyFuncModel):
|
|
os.kill(model.pid, signal.SIGTERM)
|
|
|
|
# if model_id is specified log metrics to the eval run and logged model
|
|
if model_id is not None:
|
|
mlflow.log_metrics(metrics=evaluate_result.metrics, dataset=data, model_id=model_id)
|
|
|
|
return evaluate_result
|