chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,40 @@
|
||||
### MLflow evaluation Examples
|
||||
|
||||
The examples in this directory demonstrate how to use the `mlflow.evaluate()` API. Specifically,
|
||||
they show how to evaluate a PyFunc model on a specified dataset using the builtin default evaluator
|
||||
and specified extra metrics, where the resulting metrics & artifacts are logged to MLflow Tracking.
|
||||
They also show how to specify validation thresholds for the resulting metrics to validate the quality
|
||||
of your model. See full list of examples below:
|
||||
|
||||
- Example `evaluate_on_binary_classifier.py` evaluates an xgboost `XGBClassifier` model on dataset loaded by
|
||||
`shap.datasets.adult`.
|
||||
- Example `evaluate_on_multiclass_classifier.py` evaluates a scikit-learn `LogisticRegression` model on dataset
|
||||
generated by `sklearn.datasets.make_classification`.
|
||||
- Example `evaluate_on_regressor.py` evaluate as scikit-learn `LinearRegression` model on dataset loaded by
|
||||
`sklearn.datasets.load_diabetes`
|
||||
- Example `evaluate_with_custom_metrics.py` evaluates a scikit-learn `LinearRegression`
|
||||
model with a custom metric function on dataset loaded by `sklearn.datasets.load_diabetes`
|
||||
- Example `evaluate_with_custom_metrics_comprehensive.py` evaluates a scikit-learn `LinearRegression` model
|
||||
with a comprehensive list of custom metric functions on dataset loaded by `sklearn.datasets.load_diabetes`
|
||||
- Example `evaluate_with_model_validation.py` trains both a candidate xgboost `XGBClassifier` model
|
||||
and a baseline `DummyClassifier` model on dataset loaded by `shap.datasets.adult`. Then, it validates
|
||||
the candidate model against specified thresholds on both builtin and extra metrics and the dummy model.
|
||||
|
||||
#### Prerequisites
|
||||
|
||||
```
|
||||
pip install scikit-learn xgboost shap>=0.40 matplotlib
|
||||
```
|
||||
|
||||
#### How to run the examples
|
||||
|
||||
Run in this directory with Python.
|
||||
|
||||
```sh
|
||||
python evaluate_on_binary_classifier.py
|
||||
python evaluate_on_multiclass_classifier.py
|
||||
python evaluate_on_regressor.py
|
||||
python evaluate_with_custom_metrics.py
|
||||
python evaluate_with_custom_metrics_comprehensive.py
|
||||
python evaluate_with_model_validation.py
|
||||
```
|
||||
@@ -0,0 +1,41 @@
|
||||
import shap
|
||||
import xgboost
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
import mlflow
|
||||
from mlflow.models import infer_signature
|
||||
|
||||
# Load the UCI Adult Dataset
|
||||
X, y = shap.datasets.adult()
|
||||
|
||||
# Split the data into training and test sets
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
|
||||
|
||||
# Fit an XGBoost binary classifier on the training data split
|
||||
model = xgboost.XGBClassifier().fit(X_train, y_train)
|
||||
|
||||
# Infer model signature
|
||||
predictions = model.predict(X_train)
|
||||
signature = infer_signature(X_train, predictions)
|
||||
|
||||
# Build the Evaluation Dataset from the test set
|
||||
eval_data = X_test
|
||||
eval_data["label"] = y_test
|
||||
|
||||
with mlflow.start_run() as run:
|
||||
# Log the XGBoost binary classifier model to MLflow
|
||||
model_info = mlflow.sklearn.log_model(
|
||||
model, name="model", signature=signature, serialization_format="cloudpickle"
|
||||
)
|
||||
|
||||
# Evaluate the logged model
|
||||
result = mlflow.evaluate(
|
||||
model_info.model_uri,
|
||||
eval_data,
|
||||
targets="label",
|
||||
model_type="classifier",
|
||||
evaluators=["default"],
|
||||
)
|
||||
|
||||
print(f"metrics:\n{result.metrics}")
|
||||
print(f"artifacts:\n{result.artifacts}")
|
||||
@@ -0,0 +1,25 @@
|
||||
from sklearn.datasets import make_classification
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
import mlflow
|
||||
|
||||
X, y = make_classification(n_samples=10000, n_classes=10, n_informative=5, random_state=1)
|
||||
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
|
||||
|
||||
with mlflow.start_run() as run:
|
||||
model = LogisticRegression(solver="liblinear").fit(X_train, y_train)
|
||||
model_info = mlflow.sklearn.log_model(model, name="model")
|
||||
result = mlflow.evaluate(
|
||||
model_info.model_uri,
|
||||
X_test,
|
||||
targets=y_test,
|
||||
model_type="classifier",
|
||||
evaluators="default",
|
||||
evaluator_config={"log_model_explainability": True, "explainability_nsamples": 1000},
|
||||
)
|
||||
|
||||
print(f"run_id={run.info.run_id}")
|
||||
print(f"metrics:\n{result.metrics}")
|
||||
print(f"artifacts:\n{result.artifacts}")
|
||||
@@ -0,0 +1,28 @@
|
||||
from sklearn.datasets import load_diabetes
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
import mlflow
|
||||
|
||||
diabetes_dataset = load_diabetes()
|
||||
|
||||
X_train, X_test, y_train, y_test = train_test_split(
|
||||
diabetes_dataset.data, diabetes_dataset.target, test_size=0.33, random_state=42
|
||||
)
|
||||
|
||||
with mlflow.start_run() as run:
|
||||
model = LinearRegression().fit(X_train, y_train)
|
||||
model_info = mlflow.sklearn.log_model(model, name="model")
|
||||
|
||||
result = mlflow.evaluate(
|
||||
model_info.model_uri,
|
||||
X_test,
|
||||
targets=y_test,
|
||||
model_type="regressor",
|
||||
evaluators="default",
|
||||
feature_names=diabetes_dataset.feature_names,
|
||||
evaluator_config={"explainability_nsamples": 1000},
|
||||
)
|
||||
|
||||
print(f"metrics:\n{result.metrics}")
|
||||
print(f"artifacts:\n{result.artifacts}")
|
||||
@@ -0,0 +1,67 @@
|
||||
import os
|
||||
|
||||
import openai
|
||||
import pandas as pd
|
||||
|
||||
import mlflow
|
||||
from mlflow.metrics import make_metric
|
||||
from mlflow.metrics.base import MetricValue, standard_aggregations
|
||||
|
||||
assert "OPENAI_API_KEY" in os.environ, "Please set the OPENAI_API_KEY environment variable."
|
||||
|
||||
|
||||
# Helper function to check if a string is valid python code
|
||||
def is_valid_python_code(code: str) -> bool:
|
||||
try:
|
||||
compile(code, "<string>", "exec")
|
||||
return True
|
||||
except SyntaxError:
|
||||
return False
|
||||
|
||||
|
||||
# Create an evaluation function that iterates through the predictions
|
||||
def eval_fn(predictions):
|
||||
scores = [int(is_valid_python_code(prediction)) for prediction in predictions]
|
||||
return MetricValue(
|
||||
scores=scores,
|
||||
aggregate_results=standard_aggregations(scores),
|
||||
)
|
||||
|
||||
|
||||
# Create an EvaluationMetric object for the python code metric
|
||||
valid_code_metric = make_metric(
|
||||
eval_fn=eval_fn, greater_is_better=False, name="valid_python_code", version="v1"
|
||||
)
|
||||
|
||||
eval_df = pd.DataFrame({
|
||||
"input": [
|
||||
"SELECT * FROM ",
|
||||
"import pandas",
|
||||
"def hello_world",
|
||||
],
|
||||
})
|
||||
|
||||
with mlflow.start_run() as run:
|
||||
system_prompt = (
|
||||
"Generate code that is less than 50 characters. Return only python code and nothing else."
|
||||
)
|
||||
logged_model = mlflow.openai.log_model(
|
||||
model="gpt-4o-mini",
|
||||
task=openai.chat.completions,
|
||||
name="model",
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": "{question}"},
|
||||
],
|
||||
)
|
||||
|
||||
results = mlflow.evaluate(
|
||||
logged_model.model_uri,
|
||||
eval_df,
|
||||
model_type="text",
|
||||
extra_metrics=[valid_code_metric],
|
||||
)
|
||||
print(results)
|
||||
|
||||
eval_table = results.tables["eval_results_table"]
|
||||
print(eval_table)
|
||||
@@ -0,0 +1,85 @@
|
||||
import os
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from sklearn.datasets import load_diabetes
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
import mlflow
|
||||
from mlflow.models import infer_signature, make_metric
|
||||
|
||||
# loading the diabetes dataset
|
||||
diabetes_dataset = load_diabetes(as_frame=True)
|
||||
|
||||
# split the dataset into train and test partitions
|
||||
X_train, X_test, y_train, y_test = train_test_split(
|
||||
diabetes_dataset.data, diabetes_dataset.target, test_size=0.2, random_state=123
|
||||
)
|
||||
|
||||
# train the model
|
||||
lin_reg = LinearRegression().fit(X_train, y_train)
|
||||
|
||||
# Infer model signature
|
||||
predictions = lin_reg.predict(X_train)
|
||||
signature = infer_signature(X_train, predictions)
|
||||
|
||||
# creating the evaluation dataframe
|
||||
eval_data = X_test.copy()
|
||||
eval_data["target"] = y_test
|
||||
|
||||
|
||||
def squared_diff_plus_one(eval_df, _builtin_metrics):
|
||||
"""
|
||||
This example custom metric function creates a metric based on the ``prediction`` and
|
||||
``target`` columns in ``eval_df`.
|
||||
"""
|
||||
return np.sum(np.abs(eval_df["prediction"] - eval_df["target"] + 1) ** 2)
|
||||
|
||||
|
||||
def sum_on_target_divided_by_two(_eval_df, builtin_metrics):
|
||||
"""
|
||||
This example custom metric function creates a metric derived from existing metrics in
|
||||
``builtin_metrics``.
|
||||
"""
|
||||
return builtin_metrics["sum_on_target"] / 2
|
||||
|
||||
|
||||
def prediction_target_scatter(eval_df, _builtin_metrics, artifacts_dir):
|
||||
"""
|
||||
This example custom artifact generates and saves a scatter plot to ``artifacts_dir`` that
|
||||
visualizes the relationship between the predictions and targets for the given model to a
|
||||
file as an image artifact.
|
||||
"""
|
||||
plt.scatter(eval_df["prediction"], eval_df["target"])
|
||||
plt.xlabel("Targets")
|
||||
plt.ylabel("Predictions")
|
||||
plt.title("Targets vs. Predictions")
|
||||
plot_path = os.path.join(artifacts_dir, "example_scatter_plot.png")
|
||||
plt.savefig(plot_path)
|
||||
return {"example_scatter_plot_artifact": plot_path}
|
||||
|
||||
|
||||
with mlflow.start_run() as run:
|
||||
model_info = mlflow.sklearn.log_model(lin_reg, name="model", signature=signature)
|
||||
result = mlflow.evaluate(
|
||||
model=model_info.model_uri,
|
||||
data=eval_data,
|
||||
targets="target",
|
||||
model_type="regressor",
|
||||
evaluators=["default"],
|
||||
extra_metrics=[
|
||||
make_metric(
|
||||
eval_fn=squared_diff_plus_one,
|
||||
greater_is_better=False,
|
||||
),
|
||||
make_metric(
|
||||
eval_fn=sum_on_target_divided_by_two,
|
||||
greater_is_better=True,
|
||||
),
|
||||
],
|
||||
custom_artifacts=[prediction_target_scatter],
|
||||
)
|
||||
|
||||
print(f"metrics:\n{result.metrics}")
|
||||
print(f"artifacts:\n{result.artifacts}")
|
||||
@@ -0,0 +1,84 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from matplotlib.figure import Figure
|
||||
from sklearn.datasets import load_diabetes
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
import mlflow
|
||||
from mlflow.models import infer_signature, make_metric
|
||||
|
||||
# loading the diabetes dataset
|
||||
diabetes_dataset = load_diabetes(as_frame=True)
|
||||
|
||||
# split the dataset into train and test partitions
|
||||
X_train, X_test, y_train, y_test = train_test_split(
|
||||
diabetes_dataset.data, diabetes_dataset.target, test_size=0.2, random_state=123
|
||||
)
|
||||
|
||||
# train the model
|
||||
lin_reg = LinearRegression().fit(X_train, y_train)
|
||||
|
||||
# Infer model signature
|
||||
predictions = lin_reg.predict(X_train)
|
||||
signature = infer_signature(X_train, predictions)
|
||||
|
||||
# creating the evaluation dataframe
|
||||
eval_data = X_test.copy()
|
||||
eval_data["target"] = y_test
|
||||
|
||||
|
||||
def custom_metric(eval_df, _builtin_metrics):
|
||||
return np.sum(np.abs(eval_df["prediction"] - eval_df["target"] + 1) ** 2)
|
||||
|
||||
|
||||
class ExampleClass:
|
||||
def __init__(self, x):
|
||||
self.x = x
|
||||
|
||||
|
||||
def custom_artifact(eval_df, builtin_metrics, _artifacts_dir):
|
||||
example_np_arr = np.array([1, 2, 3])
|
||||
example_df = pd.DataFrame({"test": [2.2, 3.1], "test2": [3, 2]})
|
||||
example_dict = {"hello": "there", "test_list": [0.1, 0.3, 4]}
|
||||
example_dict.update(builtin_metrics)
|
||||
example_dict_2 = '{"a": 3, "b": [1, 2, 3]}'
|
||||
example_image = Figure()
|
||||
ax = example_image.subplots()
|
||||
ax.scatter(eval_df["prediction"], eval_df["target"])
|
||||
ax.set_xlabel("Targets")
|
||||
ax.set_ylabel("Predictions")
|
||||
ax.set_title("Targets vs. Predictions")
|
||||
example_custom_class = ExampleClass(10)
|
||||
|
||||
return {
|
||||
"example_np_arr_from_obj_saved_as_npy": example_np_arr,
|
||||
"example_df_from_obj_saved_as_csv": example_df,
|
||||
"example_dict_from_obj_saved_as_json": example_dict,
|
||||
"example_image_from_obj_saved_as_png": example_image,
|
||||
"example_dict_from_json_str_saved_as_json": example_dict_2,
|
||||
"example_class_from_obj_saved_as_pickle": example_custom_class,
|
||||
}
|
||||
|
||||
|
||||
with mlflow.start_run() as run:
|
||||
model_info = mlflow.sklearn.log_model(lin_reg, name="model", signature=signature)
|
||||
result = mlflow.evaluate(
|
||||
model=model_info.model_uri,
|
||||
data=eval_data,
|
||||
targets="target",
|
||||
model_type="regressor",
|
||||
evaluators=["default"],
|
||||
extra_metrics=[
|
||||
make_metric(
|
||||
eval_fn=custom_metric,
|
||||
greater_is_better=False,
|
||||
)
|
||||
],
|
||||
custom_artifacts=[
|
||||
custom_artifact,
|
||||
],
|
||||
)
|
||||
|
||||
print(f"metrics:\n{result.metrics}")
|
||||
print(f"artifacts:\n{result.artifacts}")
|
||||
@@ -0,0 +1,37 @@
|
||||
import shap
|
||||
import xgboost
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
import mlflow
|
||||
|
||||
# Load the UCI Adult Dataset
|
||||
X, y = shap.datasets.adult()
|
||||
|
||||
# Split the data into training and test sets
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
|
||||
|
||||
# Fit an XGBoost binary classifier on the training data split
|
||||
model = xgboost.XGBClassifier().fit(X_train, y_train)
|
||||
|
||||
# Build the Evaluation Dataset from the test set
|
||||
eval_data = X_test
|
||||
eval_data["label"] = y_test
|
||||
|
||||
|
||||
# Define a function that calls the model's predict method
|
||||
def fn(X):
|
||||
return model.predict(X)
|
||||
|
||||
|
||||
with mlflow.start_run() as run:
|
||||
# Evaluate the function without logging the model
|
||||
result = mlflow.evaluate(
|
||||
fn,
|
||||
eval_data,
|
||||
targets="label",
|
||||
model_type="classifier",
|
||||
evaluators=["default"],
|
||||
)
|
||||
|
||||
print(f"metrics:\n{result.metrics}")
|
||||
print(f"artifacts:\n{result.artifacts}")
|
||||
@@ -0,0 +1,67 @@
|
||||
import os
|
||||
|
||||
import openai
|
||||
import pandas as pd
|
||||
|
||||
import mlflow
|
||||
from mlflow.metrics.genai import EvaluationExample, answer_similarity
|
||||
|
||||
assert "OPENAI_API_KEY" in os.environ, "Please set the OPENAI_API_KEY environment variable."
|
||||
|
||||
|
||||
# testing with OpenAI gpt-4o-mini
|
||||
example = EvaluationExample(
|
||||
input="What is MLflow?",
|
||||
output="MLflow is an open-source platform for managing machine "
|
||||
"learning workflows, including experiment tracking, model packaging, "
|
||||
"versioning, and deployment, simplifying the ML lifecycle.",
|
||||
score=4,
|
||||
justification="The definition effectively explains what MLflow is "
|
||||
"its purpose, and its developer. It could be more concise for a 5-score.",
|
||||
grading_context={
|
||||
"ground_truth": "MLflow is an open-source platform for managing "
|
||||
"the end-to-end machine learning (ML) lifecycle. It was developed by Databricks, "
|
||||
"a company that specializes in big data and machine learning solutions. MLflow is "
|
||||
"designed to address the challenges that data scientists and machine learning "
|
||||
"engineers face when developing, training, and deploying machine learning models."
|
||||
},
|
||||
)
|
||||
|
||||
answer_similarity_metric = answer_similarity(examples=[example])
|
||||
|
||||
eval_df = pd.DataFrame({
|
||||
"inputs": [
|
||||
"What is MLflow?",
|
||||
"What is Spark?",
|
||||
"What is Python?",
|
||||
],
|
||||
"ground_truth": [
|
||||
"MLflow is an open-source platform for managing the end-to-end machine learning (ML) lifecycle. It was developed by Databricks, a company that specializes in big data and machine learning solutions. MLflow is designed to address the challenges that data scientists and machine learning engineers face when developing, training, and deploying machine learning models.",
|
||||
"Apache Spark is an open-source, distributed computing system designed for big data processing and analytics. It was developed in response to limitations of the Hadoop MapReduce computing model, offering improvements in speed and ease of use. Spark provides libraries for various tasks such as data ingestion, processing, and analysis through its components like Spark SQL for structured data, Spark Streaming for real-time data processing, and MLlib for machine learning tasks",
|
||||
"Python is a high-level programming language that was created by Guido van Rossum and released in 1991. It emphasizes code readability and allows developers to express concepts in fewer lines of code than languages like C++ or Java. Python is used in various domains, including web development, scientific computing, data analysis, and machine learning.",
|
||||
],
|
||||
})
|
||||
|
||||
with mlflow.start_run() as run:
|
||||
system_prompt = "Answer the following question in two sentences"
|
||||
logged_model = mlflow.openai.log_model(
|
||||
model="gpt-4o-mini",
|
||||
task=openai.chat.completions,
|
||||
name="model",
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": "{question}"},
|
||||
],
|
||||
)
|
||||
|
||||
results = mlflow.evaluate(
|
||||
logged_model.model_uri,
|
||||
eval_df,
|
||||
targets="ground_truth",
|
||||
model_type="question-answering",
|
||||
extra_metrics=[answer_similarity_metric],
|
||||
)
|
||||
print(results)
|
||||
|
||||
eval_table = results.tables["eval_results_table"]
|
||||
print(eval_table)
|
||||
@@ -0,0 +1,110 @@
|
||||
import shap
|
||||
import xgboost
|
||||
from sklearn.dummy import DummyClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
import mlflow
|
||||
from mlflow.models import MetricThreshold, infer_signature, make_metric
|
||||
|
||||
# load UCI Adult Data Set; segment it into training and test sets
|
||||
X, y = shap.datasets.adult()
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
|
||||
|
||||
# train a candidate XGBoost model
|
||||
candidate_model = xgboost.XGBClassifier().fit(X_train, y_train)
|
||||
candidate_signature = infer_signature(X_train, candidate_model.predict(X_train))
|
||||
|
||||
# train a baseline dummy model
|
||||
baseline_model = DummyClassifier(strategy="uniform").fit(X_train, y_train)
|
||||
baseline_signature = infer_signature(X_train, baseline_model.predict(X_train))
|
||||
|
||||
# construct an evaluation dataset from the test set
|
||||
eval_data = X_test
|
||||
eval_data["label"] = y_test
|
||||
|
||||
|
||||
# Define a custom metric to evaluate against
|
||||
def double_positive(_eval_df, builtin_metrics):
|
||||
return builtin_metrics["true_positives"] * 2
|
||||
|
||||
|
||||
# Define criteria for model to be validated against
|
||||
thresholds = {
|
||||
# Specify metric value threshold
|
||||
"precision_score": MetricThreshold(
|
||||
threshold=0.7, greater_is_better=True
|
||||
), # precision should be >=0.7
|
||||
# Specify model comparison thresholds
|
||||
"recall_score": MetricThreshold(
|
||||
min_absolute_change=0.1, # recall should be at least 0.1 greater than baseline model recall
|
||||
min_relative_change=0.1, # recall should be at least 10 percent greater than baseline model recall
|
||||
greater_is_better=True,
|
||||
),
|
||||
# Specify both metric value and model comparison thresholds
|
||||
"accuracy_score": MetricThreshold(
|
||||
threshold=0.8, # accuracy should be >=0.8
|
||||
min_absolute_change=0.05, # accuracy should be at least 0.05 greater than baseline model accuracy
|
||||
min_relative_change=0.05, # accuracy should be at least 5 percent greater than baseline model accuracy
|
||||
greater_is_better=True,
|
||||
),
|
||||
# Specify threshold for custom metric
|
||||
"double_positive": MetricThreshold(
|
||||
threshold=1e5,
|
||||
greater_is_better=False, # double_positive should be <=1e5
|
||||
),
|
||||
}
|
||||
|
||||
double_positive_metric = make_metric(
|
||||
eval_fn=double_positive,
|
||||
greater_is_better=False,
|
||||
)
|
||||
|
||||
with mlflow.start_run() as run:
|
||||
# Note: in most model validation use-cases the baseline model should instead b
|
||||
# a previously trained model (such as the current production model)
|
||||
baseline_model_uri = mlflow.sklearn.log_model(
|
||||
baseline_model,
|
||||
name="baseline_model",
|
||||
signature=baseline_signature,
|
||||
serialization_format="cloudpickle",
|
||||
).model_uri
|
||||
|
||||
# Evaluate the baseline model
|
||||
baseline_result = mlflow.evaluate(
|
||||
baseline_model_uri,
|
||||
eval_data,
|
||||
targets="label",
|
||||
model_type="classifier",
|
||||
extra_metrics=[double_positive_metric],
|
||||
# set to env_manager to "virtualenv" or "conda" to score the candidate and baseline models
|
||||
# in isolated Python environments where their dependencies are restored.
|
||||
env_manager="local",
|
||||
)
|
||||
|
||||
# Evaluate the candidate model
|
||||
candidate_model_uri = mlflow.sklearn.log_model(
|
||||
candidate_model,
|
||||
name="candidate_model",
|
||||
signature=candidate_signature,
|
||||
serialization_format="cloudpickle",
|
||||
).model_uri
|
||||
|
||||
candidate_result = mlflow.evaluate(
|
||||
candidate_model_uri,
|
||||
eval_data,
|
||||
targets="label",
|
||||
model_type="classifier",
|
||||
extra_metrics=[double_positive_metric],
|
||||
env_manager="local",
|
||||
)
|
||||
|
||||
|
||||
# Validate the candidate result against the baseline
|
||||
mlflow.validate_evaluation_results(
|
||||
candidate_result=candidate_result,
|
||||
baseline_result=baseline_result,
|
||||
validation_thresholds=thresholds,
|
||||
)
|
||||
# If you would like to catch model validation failures, you can add try except clauses around
|
||||
# the mlflow.evaluate() call and catch the ModelValidationFailedException, imported at the top
|
||||
# of this file.
|
||||
@@ -0,0 +1,41 @@
|
||||
import openai
|
||||
import pandas as pd
|
||||
|
||||
import mlflow
|
||||
|
||||
eval_df = pd.DataFrame({
|
||||
"inputs": [
|
||||
"What is MLflow?",
|
||||
"What is Spark?",
|
||||
"What is Python?",
|
||||
],
|
||||
"ground_truth": [
|
||||
"MLflow is an open-source platform for managing the end-to-end machine learning (ML) lifecycle. It was developed by Databricks, a company that specializes in big data and machine learning solutions. MLflow is designed to address the challenges that data scientists and machine learning engineers face when developing, training, and deploying machine learning models.",
|
||||
"Apache Spark is an open-source, distributed computing system designed for big data processing and analytics. It was developed in response to limitations of the Hadoop MapReduce computing model, offering improvements in speed and ease of use. Spark provides libraries for various tasks such as data ingestion, processing, and analysis through its components like Spark SQL for structured data, Spark Streaming for real-time data processing, and MLlib for machine learning tasks",
|
||||
"Python is a high-level programming language that was created by Guido van Rossum and released in 1991. It emphasizes code readability and allows developers to express concepts in fewer lines of code than languages like C++ or Java. Python is used in various domains, including web development, scientific computing, data analysis, and machine learning.",
|
||||
],
|
||||
})
|
||||
|
||||
with mlflow.start_run() as run:
|
||||
system_prompt = "Answer the following question in two sentences"
|
||||
logged_model = mlflow.openai.log_model(
|
||||
model="gpt-4o-mini",
|
||||
task=openai.chat.completions,
|
||||
name="model",
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": "{question}"},
|
||||
],
|
||||
)
|
||||
|
||||
results = mlflow.evaluate(
|
||||
logged_model.model_uri,
|
||||
eval_df,
|
||||
targets="ground_truth",
|
||||
model_type="question-answering",
|
||||
evaluators="default",
|
||||
)
|
||||
print(results.metrics)
|
||||
|
||||
eval_table = results.tables["eval_results_table"]
|
||||
print(eval_table)
|
||||
@@ -0,0 +1,33 @@
|
||||
import shap
|
||||
import xgboost
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
import mlflow
|
||||
|
||||
# Load the UCI Adult Dataset
|
||||
X, y = shap.datasets.adult()
|
||||
|
||||
# Split the data into training and test sets
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
|
||||
|
||||
# Fit an XGBoost binary classifier on the training data split
|
||||
model = xgboost.XGBClassifier().fit(X_train, y_train)
|
||||
|
||||
# Build the Evaluation Dataset from the test set
|
||||
y_test_pred = model.predict(X=X_test)
|
||||
eval_data = X_test
|
||||
eval_data["label"] = y_test
|
||||
eval_data["predictions"] = y_test_pred
|
||||
|
||||
|
||||
with mlflow.start_run() as run:
|
||||
# Evaluate the static dataset without providing a model
|
||||
result = mlflow.evaluate(
|
||||
data=eval_data,
|
||||
targets="label",
|
||||
predictions="predictions",
|
||||
model_type="classifier",
|
||||
)
|
||||
|
||||
print(f"metrics:\n{result.metrics}")
|
||||
print(f"artifacts:\n{result.artifacts}")
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,590 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {},
|
||||
"inputWidgets": {},
|
||||
"nuid": "42084110-295b-493a-9b3e-5d8d29ff78b3",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"# LLM RAG Evaluation with MLflow Example Notebook\n",
|
||||
"\n",
|
||||
"In this notebook, we will demonstrate how to evaluate various a RAG system with MLflow."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {},
|
||||
"inputWidgets": {},
|
||||
"nuid": "bdff35e3-0e09-48b8-87ce-78759de88998",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"We need to set our OpenAI API key, since we will be using GPT-4 for our LLM-judged metrics.\n",
|
||||
"\n",
|
||||
"In order to set your private key safely, please be sure to either export your key through a command-line terminal for your current instance, or, for a permanent addition to all user-based sessions, configure your favored environment management configuration file (i.e., .bashrc, .zshrc) to have the following entry:\n",
|
||||
"\n",
|
||||
"`OPENAI_API_KEY=<your openai API key>`"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {
|
||||
"byteLimit": 2048000,
|
||||
"rowLimit": 10000
|
||||
},
|
||||
"inputWidgets": {},
|
||||
"nuid": "fb946228-62fb-4d68-9732-75935c9cb401",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"import mlflow"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {},
|
||||
"inputWidgets": {},
|
||||
"nuid": "273d1345-95d7-435a-a7b6-a5f3dbb3f073",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"## Create a RAG system\n",
|
||||
"\n",
|
||||
"Use Langchain and Chroma to create a RAG system that answers questions based on the MLflow documentation."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {
|
||||
"byteLimit": 2048000,
|
||||
"rowLimit": 10000
|
||||
},
|
||||
"inputWidgets": {},
|
||||
"nuid": "2c28d0ad-f469-46ab-a2b4-c5e8db50a729",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chains import RetrievalQA\n",
|
||||
"from langchain.document_loaders import WebBaseLoader\n",
|
||||
"from langchain.embeddings.openai import OpenAIEmbeddings\n",
|
||||
"from langchain.llms import OpenAI\n",
|
||||
"from langchain.text_splitter import CharacterTextSplitter\n",
|
||||
"from langchain.vectorstores import Chroma"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {
|
||||
"byteLimit": 2048000,
|
||||
"rowLimit": 10000
|
||||
},
|
||||
"inputWidgets": {},
|
||||
"nuid": "83a7e77e-6717-472a-86dc-02e2c356ddef",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"loader = WebBaseLoader(\"https://mlflow.org/docs/latest/index.html\")\n",
|
||||
"\n",
|
||||
"documents = loader.load()\n",
|
||||
"text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
|
||||
"texts = text_splitter.split_documents(documents)\n",
|
||||
"\n",
|
||||
"embeddings = OpenAIEmbeddings()\n",
|
||||
"docsearch = Chroma.from_documents(texts, embeddings)\n",
|
||||
"\n",
|
||||
"qa = RetrievalQA.from_chain_type(\n",
|
||||
" llm=OpenAI(temperature=0),\n",
|
||||
" chain_type=\"stuff\",\n",
|
||||
" retriever=docsearch.as_retriever(),\n",
|
||||
" return_source_documents=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {},
|
||||
"inputWidgets": {},
|
||||
"nuid": "fd70bcf6-7c44-44d3-9435-567b82611e1c",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"## Evaluate the RAG system using `mlflow.evaluate()`"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {},
|
||||
"inputWidgets": {},
|
||||
"nuid": "de1bc359-2e40-459c-bea4-bed35a117988",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"Create a simple function that runs each input through the RAG chain"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {
|
||||
"byteLimit": 2048000,
|
||||
"rowLimit": 10000
|
||||
},
|
||||
"inputWidgets": {},
|
||||
"nuid": "667ec809-2bb5-4170-9937-6804386b41ec",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def model(input_df):\n",
|
||||
" answer = []\n",
|
||||
" for index, row in input_df.iterrows():\n",
|
||||
" answer.append(qa(row[\"questions\"]))\n",
|
||||
"\n",
|
||||
" return answer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {},
|
||||
"inputWidgets": {},
|
||||
"nuid": "d1064306-b7f3-4b3e-825c-4353d808f21d",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"Create an eval dataset"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {
|
||||
"byteLimit": 2048000,
|
||||
"rowLimit": 10000
|
||||
},
|
||||
"inputWidgets": {},
|
||||
"nuid": "a5481491-e4a9-42ea-8a3f-f527faffd04d",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"eval_df = pd.DataFrame({\n",
|
||||
" \"questions\": [\n",
|
||||
" \"What is MLflow?\",\n",
|
||||
" \"How to run mlflow.evaluate()?\",\n",
|
||||
" \"How to log_table()?\",\n",
|
||||
" \"How to load_table()?\",\n",
|
||||
" ],\n",
|
||||
"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {},
|
||||
"inputWidgets": {},
|
||||
"nuid": "9c3c8023-8feb-427a-b36d-34cd1853a5dc",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"Create a faithfulness metric"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {
|
||||
"byteLimit": 2048000,
|
||||
"rowLimit": 10000
|
||||
},
|
||||
"inputWidgets": {},
|
||||
"nuid": "3882b940-9c25-41ce-a301-72d8c0c90aaa",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from mlflow.metrics.genai.metric_definitions import faithfulness\n",
|
||||
"\n",
|
||||
"faithfulness_metric = faithfulness(model=\"openai:/gpt-4\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {
|
||||
"byteLimit": 2048000,
|
||||
"rowLimit": 10000
|
||||
},
|
||||
"inputWidgets": {},
|
||||
"nuid": "ea40ce52-6ac7-4c20-9669-d24f80a6cebe",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"2023/10/23 13:13:16 INFO mlflow.models.evaluation.base: Evaluating the model with the default evaluator.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Number of requested results 4 is greater than number of elements in index 3, updating n_results = 3\n",
|
||||
"Number of requested results 4 is greater than number of elements in index 3, updating n_results = 3\n",
|
||||
"Number of requested results 4 is greater than number of elements in index 3, updating n_results = 3\n",
|
||||
"Number of requested results 4 is greater than number of elements in index 3, updating n_results = 3\n",
|
||||
"Using pad_token, but it is not set yet.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "23e9a5f58f1b4930ac47c88259156e1d",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/1 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"2023/10/23 13:13:41 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: token_count\n",
|
||||
"2023/10/23 13:13:41 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: toxicity\n",
|
||||
"2023/10/23 13:13:41 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: perplexity\n",
|
||||
"Using pad_token, but it is not set yet.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "2c6fd2067bad4404ad5550d56e23407e",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/1 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"2023/10/23 13:13:44 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: flesch_kincaid_grade_level\n",
|
||||
"2023/10/23 13:13:44 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: ari_grade_level\n",
|
||||
"2023/10/23 13:13:44 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: exact_match\n",
|
||||
"2023/10/23 13:13:44 INFO mlflow.models.evaluation.default_evaluator: Evaluating metrics: faithfulness\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'toxicity/v1/mean': 0.0002736186215770431, 'toxicity/v1/variance': 2.856656765360073e-08, 'toxicity/v1/p90': 0.0004570253004203551, 'toxicity/v1/ratio': 0.0, 'perplexity/v1/mean': 70.08646988868713, 'perplexity/v1/variance': 5233.465638493719, 'perplexity/v1/p90': 149.10144042968753, 'flesch_kincaid_grade_level/v1/mean': 7.625, 'flesch_kincaid_grade_level/v1/variance': 23.836875, 'flesch_kincaid_grade_level/v1/p90': 13.150000000000002, 'ari_grade_level/v1/mean': 9.450000000000001, 'ari_grade_level/v1/variance': 32.262499999999996, 'ari_grade_level/v1/p90': 15.870000000000001, 'faithfulness/v1/mean': 4.0, 'faithfulness/v1/variance': 3.0, 'faithfulness/v1/p90': 5.0}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"results = mlflow.evaluate(\n",
|
||||
" model,\n",
|
||||
" eval_df,\n",
|
||||
" model_type=\"question-answering\",\n",
|
||||
" evaluators=\"default\",\n",
|
||||
" predictions=\"result\",\n",
|
||||
" extra_metrics=[faithfulness_metric, mlflow.metrics.latency()],\n",
|
||||
" evaluator_config={\n",
|
||||
" \"col_mapping\": {\n",
|
||||
" \"inputs\": \"questions\",\n",
|
||||
" \"context\": \"source_documents\",\n",
|
||||
" }\n",
|
||||
" },\n",
|
||||
")\n",
|
||||
"print(results.metrics)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {},
|
||||
"inputWidgets": {},
|
||||
"nuid": "989a0861-5153-44e6-a19d-efcae7fe6cb5",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "4a4883be06c94983a171da51d14b40a3",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading artifacts: 0%| | 0/1 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>questions</th>\n",
|
||||
" <th>outputs</th>\n",
|
||||
" <th>query</th>\n",
|
||||
" <th>source_documents</th>\n",
|
||||
" <th>latency</th>\n",
|
||||
" <th>token_count</th>\n",
|
||||
" <th>toxicity/v1/score</th>\n",
|
||||
" <th>perplexity/v1/score</th>\n",
|
||||
" <th>flesch_kincaid_grade_level/v1/score</th>\n",
|
||||
" <th>ari_grade_level/v1/score</th>\n",
|
||||
" <th>faithfulness/v1/score</th>\n",
|
||||
" <th>faithfulness/v1/justification</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>What is MLflow?</td>\n",
|
||||
" <td>MLflow is an open source platform for managin...</td>\n",
|
||||
" <td>What is MLflow?</td>\n",
|
||||
" <td>[{'lc_attributes': {}, 'lc_namespace': ['langc...</td>\n",
|
||||
" <td>3.970739</td>\n",
|
||||
" <td>176</td>\n",
|
||||
" <td>0.000208</td>\n",
|
||||
" <td>28.626591</td>\n",
|
||||
" <td>15.4</td>\n",
|
||||
" <td>18.9</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" <td>The output provided by the model is a detailed...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>How to run MLflow.evaluate()?</td>\n",
|
||||
" <td>\\n\\nYou can run MLflow.evaluate() by using the...</td>\n",
|
||||
" <td>How to run MLflow.evaluate()?</td>\n",
|
||||
" <td>[{'lc_attributes': {}, 'lc_namespace': ['langc...</td>\n",
|
||||
" <td>1.083653</td>\n",
|
||||
" <td>39</td>\n",
|
||||
" <td>0.000179</td>\n",
|
||||
" <td>44.533493</td>\n",
|
||||
" <td>4.7</td>\n",
|
||||
" <td>4.5</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" <td>The output states that \"You can run MLflow.eva...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>How to log_table()?</td>\n",
|
||||
" <td>\\n\\nYou can use the log_table() function in ML...</td>\n",
|
||||
" <td>How to log_table()?</td>\n",
|
||||
" <td>[{'lc_attributes': {}, 'lc_namespace': ['langc...</td>\n",
|
||||
" <td>2.833117</td>\n",
|
||||
" <td>114</td>\n",
|
||||
" <td>0.000564</td>\n",
|
||||
" <td>13.269521</td>\n",
|
||||
" <td>7.9</td>\n",
|
||||
" <td>8.8</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>The output provides a detailed explanation of ...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>How to load_table()?</td>\n",
|
||||
" <td>load_table() is not a function in MLflow.</td>\n",
|
||||
" <td>How to load_table()?</td>\n",
|
||||
" <td>[{'lc_attributes': {}, 'lc_namespace': ['langc...</td>\n",
|
||||
" <td>3.736170</td>\n",
|
||||
" <td>11</td>\n",
|
||||
" <td>0.000144</td>\n",
|
||||
" <td>193.916275</td>\n",
|
||||
" <td>2.5</td>\n",
|
||||
" <td>5.6</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" <td>The output states that \"load_table() is not a ...</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" questions \\\n",
|
||||
"0 What is MLflow? \n",
|
||||
"1 How to run MLflow.evaluate()? \n",
|
||||
"2 How to log_table()? \n",
|
||||
"3 How to load_table()? \n",
|
||||
"\n",
|
||||
" outputs \\\n",
|
||||
"0 MLflow is an open source platform for managin... \n",
|
||||
"1 \\n\\nYou can run MLflow.evaluate() by using the... \n",
|
||||
"2 \\n\\nYou can use the log_table() function in ML... \n",
|
||||
"3 load_table() is not a function in MLflow. \n",
|
||||
"\n",
|
||||
" query \\\n",
|
||||
"0 What is MLflow? \n",
|
||||
"1 How to run MLflow.evaluate()? \n",
|
||||
"2 How to log_table()? \n",
|
||||
"3 How to load_table()? \n",
|
||||
"\n",
|
||||
" source_documents latency token_count \\\n",
|
||||
"0 [{'lc_attributes': {}, 'lc_namespace': ['langc... 3.970739 176 \n",
|
||||
"1 [{'lc_attributes': {}, 'lc_namespace': ['langc... 1.083653 39 \n",
|
||||
"2 [{'lc_attributes': {}, 'lc_namespace': ['langc... 2.833117 114 \n",
|
||||
"3 [{'lc_attributes': {}, 'lc_namespace': ['langc... 3.736170 11 \n",
|
||||
"\n",
|
||||
" toxicity/v1/score perplexity/v1/score \\\n",
|
||||
"0 0.000208 28.626591 \n",
|
||||
"1 0.000179 44.533493 \n",
|
||||
"2 0.000564 13.269521 \n",
|
||||
"3 0.000144 193.916275 \n",
|
||||
"\n",
|
||||
" flesch_kincaid_grade_level/v1/score ari_grade_level/v1/score \\\n",
|
||||
"0 15.4 18.9 \n",
|
||||
"1 4.7 4.5 \n",
|
||||
"2 7.9 8.8 \n",
|
||||
"3 2.5 5.6 \n",
|
||||
"\n",
|
||||
" faithfulness/v1/score faithfulness/v1/justification \n",
|
||||
"0 5 The output provided by the model is a detailed... \n",
|
||||
"1 5 The output states that \"You can run MLflow.eva... \n",
|
||||
"2 1 The output provides a detailed explanation of ... \n",
|
||||
"3 5 The output states that \"load_table() is not a ... "
|
||||
]
|
||||
},
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"results.tables[\"eval_results_table\"]"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+notebook": {
|
||||
"dashboards": [],
|
||||
"language": "python",
|
||||
"notebookMetadata": {
|
||||
"pythonIndentUnit": 2
|
||||
},
|
||||
"notebookName": "LLM Evaluation Examples -- RAG",
|
||||
"widgets": {}
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "mlflow-dev-env",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.17"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
Reference in New Issue
Block a user