chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,691 @@
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from numbers import Number
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import pyarrow.fs
|
||||
|
||||
from ray.air.constants import EXPR_PROGRESS_FILE, EXPR_RESULT_FILE, TRAINING_ITERATION
|
||||
from ray.train._internal.storage import _exists_at_fs_path, get_fs_and_path
|
||||
from ray.tune import Checkpoint
|
||||
from ray.tune.execution.experiment_state import _find_newest_experiment_checkpoint
|
||||
from ray.tune.execution.tune_controller import TuneController
|
||||
from ray.tune.experiment import Trial
|
||||
from ray.tune.result import CONFIG_PREFIX, DEFAULT_METRIC
|
||||
from ray.tune.utils import flatten_dict
|
||||
from ray.tune.utils.serialization import _loads_with_cloudpickle
|
||||
from ray.tune.utils.util import is_nan, is_nan_or_inf, unflattened_lookup
|
||||
from ray.util.annotations import PublicAPI
|
||||
|
||||
try:
|
||||
import pandas as pd
|
||||
from pandas import DataFrame
|
||||
except ImportError:
|
||||
pd = None
|
||||
DataFrame = None
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@PublicAPI(stability="beta")
|
||||
class ExperimentAnalysis:
|
||||
"""Analyze results from a Ray Train/Tune experiment.
|
||||
|
||||
To use this class, the run must store the history of reported metrics
|
||||
in log files (e.g., `result.json` and `progress.csv`).
|
||||
This is the default behavior, unless default loggers are explicitly excluded
|
||||
with the `TUNE_DISABLE_AUTO_CALLBACK_LOGGERS=1` environment variable.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
experiment_checkpoint_path: Union[str, os.PathLike],
|
||||
*,
|
||||
storage_filesystem: Optional[pyarrow.fs.FileSystem] = None,
|
||||
trials: Optional[List[Trial]] = None,
|
||||
default_metric: Optional[str] = None,
|
||||
default_mode: Optional[str] = None,
|
||||
):
|
||||
"""Initialize an ``ExperimentAnalysis``.
|
||||
|
||||
Args:
|
||||
experiment_checkpoint_path: Path to an `experiment_state.json` file,
|
||||
or a directory that contains an `experiment_state.json` file.
|
||||
storage_filesystem: A custom ``pyarrow.fs.FileSystem`` corresponding
|
||||
to ``experiment_checkpoint_path``. This may be necessary if the
|
||||
original experiment used a custom filesystem.
|
||||
trials: List of trials that can be accessed via `analysis.trials`.
|
||||
default_metric: Default metric for comparing results. Can be
|
||||
overwritten with the ``metric`` parameter in the respective
|
||||
functions.
|
||||
default_mode: Default mode for comparing results. Has to be one
|
||||
of [min, max]. Can be overwritten with the ``mode`` parameter
|
||||
in the respective functions.
|
||||
"""
|
||||
self.default_metric = default_metric
|
||||
if default_mode and default_mode not in ["min", "max"]:
|
||||
raise ValueError("`default_mode` has to be None or one of [min, max]")
|
||||
self.default_mode = default_mode
|
||||
if self.default_metric is None and self.default_mode is not None:
|
||||
# If only a mode was passed, use anonymous metric
|
||||
self.default_metric = DEFAULT_METRIC
|
||||
|
||||
# Resolve the filesystem if not specified.
|
||||
if storage_filesystem:
|
||||
self._fs = storage_filesystem
|
||||
else:
|
||||
self._fs, experiment_checkpoint_path = get_fs_and_path(
|
||||
experiment_checkpoint_path
|
||||
)
|
||||
|
||||
# Find the json state file.
|
||||
experiment_checkpoint_path = str(experiment_checkpoint_path)
|
||||
if experiment_checkpoint_path.endswith(".json"):
|
||||
self._experiment_fs_path = os.path.dirname(experiment_checkpoint_path)
|
||||
self._experiment_json_fs_path = experiment_checkpoint_path
|
||||
else:
|
||||
self._experiment_fs_path = experiment_checkpoint_path
|
||||
|
||||
experiment_json_fs_path = _find_newest_experiment_checkpoint(
|
||||
experiment_path=self._experiment_fs_path, fs=self._fs
|
||||
)
|
||||
if experiment_json_fs_path is None:
|
||||
pattern = TuneController.CKPT_FILE_TMPL.format("*")
|
||||
raise ValueError(
|
||||
f"No experiment snapshot file of form '{pattern}' was found at: "
|
||||
f"({self._fs.type_name}, {self._experiment_fs_path})\n"
|
||||
"Please check if you specified the correct experiment path, "
|
||||
"which should be a combination of the `storage_path` and `name` "
|
||||
"specified in your run."
|
||||
)
|
||||
|
||||
self._experiment_json_fs_path = experiment_json_fs_path
|
||||
|
||||
self.trials = trials or self._load_trials()
|
||||
self._trial_dataframes = self._fetch_trial_dataframes()
|
||||
self._configs = self.get_all_configs()
|
||||
|
||||
def _load_trials(self) -> List[Trial]:
|
||||
with self._fs.open_input_stream(self._experiment_json_fs_path) as f:
|
||||
experiment_state = _loads_with_cloudpickle(f.readall())
|
||||
|
||||
experiment_fs_path = Path(self._experiment_fs_path)
|
||||
|
||||
trials = []
|
||||
trial_states = experiment_state["trial_data"]
|
||||
for trial_json_state, trial_runtime_metadata in trial_states:
|
||||
trial = Trial.from_json_state(trial_json_state, stub=True)
|
||||
trial.restore_run_metadata(trial_runtime_metadata)
|
||||
|
||||
new_storage = copy.copy(trial.storage)
|
||||
new_storage.storage_fs_path = experiment_fs_path.parent.as_posix()
|
||||
new_storage.storage_filesystem = self._fs
|
||||
new_storage.experiment_dir_name = experiment_fs_path.name
|
||||
trial.set_storage(new_storage)
|
||||
|
||||
trials.append(trial)
|
||||
return trials
|
||||
|
||||
def _fetch_trial_dataframe(self, trial: Trial) -> DataFrame:
|
||||
force_dtype = {"trial_id": str} # Never convert trial_id to float.
|
||||
|
||||
# If there were no reported results, there will be no files into a DataFrame
|
||||
if trial.last_result is None:
|
||||
return DataFrame()
|
||||
|
||||
json_fs_path = Path(trial.storage.trial_fs_path, EXPR_RESULT_FILE).as_posix()
|
||||
csv_fs_path = Path(trial.storage.trial_fs_path, EXPR_PROGRESS_FILE).as_posix()
|
||||
# Prefer reading the JSON if it exists.
|
||||
if _exists_at_fs_path(trial.storage.storage_filesystem, json_fs_path):
|
||||
with trial.storage.storage_filesystem.open_input_stream(json_fs_path) as f:
|
||||
content = f.readall().decode("utf-8").rstrip("\n")
|
||||
if not content:
|
||||
return DataFrame()
|
||||
json_list = [json.loads(row) for row in content.split("\n")]
|
||||
df = pd.json_normalize(json_list, sep="/")
|
||||
# Fallback to reading the CSV.
|
||||
elif _exists_at_fs_path(trial.storage.storage_filesystem, csv_fs_path):
|
||||
with trial.storage.storage_filesystem.open_input_stream(csv_fs_path) as f:
|
||||
csv_str = f.readall().decode("utf-8")
|
||||
df = pd.read_csv(io.StringIO(csv_str), dtype=force_dtype)
|
||||
else:
|
||||
raise FileNotFoundError(
|
||||
f"Could not fetch metrics for {trial}: both {EXPR_RESULT_FILE} and "
|
||||
f"{EXPR_PROGRESS_FILE} were not found at {trial.storage.trial_fs_path}"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
def _fetch_trial_dataframes(self) -> Dict[str, DataFrame]:
|
||||
"""Fetches trial dataframes from files.
|
||||
|
||||
Returns:
|
||||
A dictionary mapping trial_id -> pd.DataFrame
|
||||
"""
|
||||
failures = []
|
||||
|
||||
trial_dfs = {}
|
||||
for trial in self.trials:
|
||||
try:
|
||||
trial_dfs[trial.trial_id] = self._fetch_trial_dataframe(trial)
|
||||
except Exception as e:
|
||||
failures.append((trial, e))
|
||||
trial_dfs[trial.trial_id] = DataFrame()
|
||||
continue
|
||||
|
||||
if failures:
|
||||
fail_str = "\n".join(
|
||||
[f"- {trial}: {repr(error)}" for trial, error in failures]
|
||||
)
|
||||
logger.warning(
|
||||
f"Failed to fetch metrics for {len(failures)} trial(s):\n{fail_str}"
|
||||
)
|
||||
return trial_dfs
|
||||
|
||||
def get_all_configs(self, prefix: bool = False) -> Dict[str, Dict]:
|
||||
"""Returns all trial hyperparameter configurations.
|
||||
|
||||
Args:
|
||||
prefix: If True, flattens the config dict
|
||||
and prepends `config/`.
|
||||
|
||||
Returns:
|
||||
Dict[str, Dict]: Mapping trial_id -> config dict
|
||||
"""
|
||||
return {
|
||||
trial.trial_id: (
|
||||
flatten_dict({CONFIG_PREFIX: trial.config}) if prefix else trial.config
|
||||
)
|
||||
for trial in self.trials
|
||||
}
|
||||
|
||||
@property
|
||||
def experiment_path(self) -> str:
|
||||
"""Path pointing to the experiment directory on persistent storage.
|
||||
|
||||
This can point to a remote storage location (e.g. S3) or to a local
|
||||
location (path on the head node)."""
|
||||
return self._experiment_fs_path
|
||||
|
||||
@property
|
||||
def best_trial(self) -> Trial:
|
||||
"""Get the best trial of the experiment
|
||||
|
||||
The best trial is determined by comparing the last trial results
|
||||
using the `metric` and `mode` parameters passed to `tune.run()`.
|
||||
|
||||
If you didn't pass these parameters, use
|
||||
`get_best_trial(metric, mode, scope)` instead.
|
||||
"""
|
||||
if not self.default_metric or not self.default_mode:
|
||||
raise ValueError(
|
||||
"To fetch the `best_trial`, pass a `metric` and `mode` "
|
||||
"parameter to `tune.run()`. Alternatively, use the "
|
||||
"`get_best_trial(metric, mode)` method to set the metric "
|
||||
"and mode explicitly."
|
||||
)
|
||||
return self.get_best_trial(self.default_metric, self.default_mode)
|
||||
|
||||
@property
|
||||
def best_config(self) -> Dict:
|
||||
"""Get the config of the best trial of the experiment
|
||||
|
||||
The best trial is determined by comparing the last trial results
|
||||
using the `metric` and `mode` parameters passed to `tune.run()`.
|
||||
|
||||
If you didn't pass these parameters, use
|
||||
`get_best_config(metric, mode, scope)` instead.
|
||||
"""
|
||||
if not self.default_metric or not self.default_mode:
|
||||
raise ValueError(
|
||||
"To fetch the `best_config`, pass a `metric` and `mode` "
|
||||
"parameter to `tune.run()`. Alternatively, use the "
|
||||
"`get_best_config(metric, mode)` method to set the metric "
|
||||
"and mode explicitly."
|
||||
)
|
||||
return self.get_best_config(self.default_metric, self.default_mode)
|
||||
|
||||
@property
|
||||
def best_checkpoint(self) -> Checkpoint:
|
||||
"""Get the checkpoint path of the best trial of the experiment
|
||||
|
||||
The best trial is determined by comparing the last trial results
|
||||
using the `metric` and `mode` parameters passed to `tune.run()`.
|
||||
|
||||
If you didn't pass these parameters, use
|
||||
`get_best_checkpoint(trial, metric, mode)` instead.
|
||||
|
||||
Returns:
|
||||
:class:`Checkpoint <ray.tune.Checkpoint>` object.
|
||||
"""
|
||||
if not self.default_metric or not self.default_mode:
|
||||
raise ValueError(
|
||||
"To fetch the `best_checkpoint`, pass a `metric` and `mode` "
|
||||
"parameter to `tune.run()`. Alternatively, use the "
|
||||
"`get_best_checkpoint(trial, metric, mode)` method to set the "
|
||||
"metric and mode explicitly."
|
||||
)
|
||||
best_trial = self.best_trial
|
||||
if not best_trial:
|
||||
raise ValueError(
|
||||
f"No best trial found. Please check if you specified the "
|
||||
f"correct default metric ({self.default_metric}) and mode "
|
||||
f"({self.default_mode})."
|
||||
)
|
||||
return self.get_best_checkpoint(
|
||||
best_trial, self.default_metric, self.default_mode
|
||||
)
|
||||
|
||||
@property
|
||||
def best_dataframe(self) -> DataFrame:
|
||||
"""Get the full result dataframe of the best trial of the experiment
|
||||
|
||||
The best trial is determined by comparing the last trial results
|
||||
using the `metric` and `mode` parameters passed to `tune.run()`.
|
||||
|
||||
If you didn't pass these parameters, use
|
||||
`get_best_trial(metric, mode)` and use it to look for the dataframe
|
||||
in the `self.trial_dataframes` dict.
|
||||
"""
|
||||
if not self.default_metric or not self.default_mode:
|
||||
raise ValueError(
|
||||
"To fetch the `best_result`, pass a `metric` and `mode` "
|
||||
"parameter to `tune.run()`."
|
||||
)
|
||||
return self.trial_dataframes[self.best_trial.trial_id]
|
||||
|
||||
@property
|
||||
def best_result(self) -> Dict:
|
||||
"""Get the last result of the best trial of the experiment
|
||||
|
||||
The best trial is determined by comparing the last trial results
|
||||
using the `metric` and `mode` parameters passed to `tune.run()`.
|
||||
|
||||
If you didn't pass these parameters, use
|
||||
`get_best_trial(metric, mode, scope).last_result` instead.
|
||||
"""
|
||||
if not self.default_metric or not self.default_mode:
|
||||
raise ValueError(
|
||||
"To fetch the `best_result`, pass a `metric` and `mode` "
|
||||
"parameter to `tune.run()`. Alternatively, use "
|
||||
"`get_best_trial(metric, mode).last_result` to set "
|
||||
"the metric and mode explicitly and fetch the last result."
|
||||
)
|
||||
return self.best_trial.last_result
|
||||
|
||||
def _delimiter(self):
|
||||
return os.environ.get("TUNE_RESULT_DELIM", "/")
|
||||
|
||||
@property
|
||||
def best_result_df(self) -> DataFrame:
|
||||
"""Get the best result of the experiment as a pandas dataframe.
|
||||
|
||||
The best trial is determined by comparing the last trial results
|
||||
using the `metric` and `mode` parameters passed to `tune.run()`.
|
||||
|
||||
If you didn't pass these parameters, use
|
||||
`get_best_trial(metric, mode, scope).last_result` instead.
|
||||
"""
|
||||
if not pd:
|
||||
raise ValueError(
|
||||
"`best_result_df` requires pandas. Install with "
|
||||
"`pip install pandas`."
|
||||
)
|
||||
|
||||
best_result = flatten_dict(self.best_result, delimiter=self._delimiter())
|
||||
return pd.DataFrame.from_records([best_result], index="trial_id")
|
||||
|
||||
@property
|
||||
def results(self) -> Dict[str, Dict]:
|
||||
"""Get the last result of the all trials of the experiment"""
|
||||
return {trial.trial_id: trial.last_result for trial in self.trials}
|
||||
|
||||
@property
|
||||
def results_df(self) -> DataFrame:
|
||||
"""Get all the last results as a pandas dataframe."""
|
||||
if not pd:
|
||||
raise ValueError(
|
||||
"`results_df` requires pandas. Install with `pip install pandas`."
|
||||
)
|
||||
return pd.DataFrame.from_records(
|
||||
[
|
||||
flatten_dict(trial.last_result, delimiter=self._delimiter())
|
||||
for trial in self.trials
|
||||
],
|
||||
index="trial_id",
|
||||
)
|
||||
|
||||
@property
|
||||
def trial_dataframes(self) -> Dict[str, DataFrame]:
|
||||
"""List of all dataframes of the trials.
|
||||
|
||||
Each dataframe is indexed by iterations and contains reported
|
||||
metrics.
|
||||
"""
|
||||
return self._trial_dataframes
|
||||
|
||||
def dataframe(
|
||||
self, metric: Optional[str] = None, mode: Optional[str] = None
|
||||
) -> DataFrame:
|
||||
"""Returns a pandas.DataFrame object constructed from the trials.
|
||||
|
||||
This function will look through all observed results of each trial
|
||||
and return the one corresponding to the passed ``metric`` and
|
||||
``mode``: If ``mode=min``, it returns the result with the lowest
|
||||
*ever* observed ``metric`` for this trial (this is not necessarily
|
||||
the last)! For ``mode=max``, it's the highest, respectively. If
|
||||
``metric=None`` or ``mode=None``, the last result will be returned.
|
||||
|
||||
Args:
|
||||
metric: Key for trial info to order on. If None, uses last result.
|
||||
mode: One of [None, "min", "max"].
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: Constructed from a result dict of each trial.
|
||||
"""
|
||||
# Do not validate metric/mode here or set from default metric/mode!
|
||||
# Otherwise we will get confusing results as the lowest ever observed
|
||||
# result may not be the last result.
|
||||
if mode and mode not in ["min", "max"]:
|
||||
raise ValueError("If set, `mode` has to be one of [min, max]")
|
||||
|
||||
if mode and not metric:
|
||||
raise ValueError(
|
||||
"If a `mode` is passed to `ExperimentAnalysis.dataframe(),"
|
||||
" you'll also have to pass a `metric`!"
|
||||
)
|
||||
|
||||
rows = self._retrieve_rows(metric=metric, mode=mode)
|
||||
all_configs = self.get_all_configs(prefix=True)
|
||||
for path, config in all_configs.items():
|
||||
if path in rows:
|
||||
rows[path].update(config)
|
||||
rows[path].update(logdir=path)
|
||||
return pd.DataFrame(list(rows.values()))
|
||||
|
||||
def _get_trial_checkpoints_with_metric(
|
||||
self, trial: Trial, metric: Optional[str] = None
|
||||
) -> List[Tuple[Checkpoint, Number]]:
|
||||
"""Get all checkpoints and a specified metric of a trial.
|
||||
|
||||
Args:
|
||||
trial: The log directory of a trial, or a trial instance.
|
||||
metric: key for trial info to return, e.g. "mean_accuracy".
|
||||
"training_iteration" is used by default if no value was
|
||||
passed to ``self.default_metric``.
|
||||
|
||||
Returns:
|
||||
List of [Checkpoint, metric] for all checkpoints of the trial.
|
||||
"""
|
||||
metric = metric or self.default_metric or TRAINING_ITERATION
|
||||
|
||||
best_checkpoint_results = (
|
||||
trial.run_metadata.checkpoint_manager.best_checkpoint_results
|
||||
)
|
||||
best_checkpoints = [
|
||||
(checkpoint_result.checkpoint, checkpoint_result.metrics)
|
||||
for checkpoint_result in best_checkpoint_results
|
||||
]
|
||||
# Support nested metrics given as flattened strings, e.g.
|
||||
# "info/learner/default_policy/policy_loss".
|
||||
return [
|
||||
(checkpoint, unflattened_lookup(metric, metrics))
|
||||
for checkpoint, metrics in best_checkpoints
|
||||
]
|
||||
|
||||
def get_best_checkpoint(
|
||||
self,
|
||||
trial: Trial,
|
||||
metric: Optional[str] = None,
|
||||
mode: Optional[str] = None,
|
||||
) -> Optional[Checkpoint]:
|
||||
"""Gets best persistent checkpoint path of provided trial.
|
||||
|
||||
Any checkpoints with an associated metric value of ``nan`` will be filtered out.
|
||||
|
||||
Args:
|
||||
trial: The log directory of a trial, or a trial instance.
|
||||
metric: key of trial info to return, e.g. "mean_accuracy".
|
||||
"training_iteration" is used by default if no value was
|
||||
passed to ``self.default_metric``.
|
||||
mode: One of [min, max]. Defaults to ``self.default_mode``.
|
||||
|
||||
Returns:
|
||||
A :class:`Checkpoint <ray.tune.Checkpoint>` object
|
||||
"""
|
||||
metric = metric or self.default_metric or TRAINING_ITERATION
|
||||
mode = self._validate_mode(mode)
|
||||
|
||||
checkpoints_and_metrics = self._get_trial_checkpoints_with_metric(trial, metric)
|
||||
|
||||
# Filter out nan. Sorting nan values leads to undefined behavior.
|
||||
checkpoints_and_metrics = list(
|
||||
filter(lambda x: not is_nan(x[1]), checkpoints_and_metrics)
|
||||
)
|
||||
|
||||
if not checkpoints_and_metrics:
|
||||
logger.error(f"No checkpoints have been found for trial {trial}.")
|
||||
return None
|
||||
|
||||
score_order_factor = -1 if mode == "min" else 1
|
||||
best_checkpoint, _ = max(
|
||||
checkpoints_and_metrics, key=lambda x: score_order_factor * x[1]
|
||||
)
|
||||
return best_checkpoint
|
||||
|
||||
def get_best_trial(
|
||||
self,
|
||||
metric: Optional[str] = None,
|
||||
mode: Optional[str] = None,
|
||||
scope: str = "last",
|
||||
filter_nan_and_inf: bool = True,
|
||||
) -> Optional[Trial]:
|
||||
"""Retrieve the best trial object.
|
||||
|
||||
Compares all trials' scores on ``metric``.
|
||||
If ``metric`` is not specified, ``self.default_metric`` will be used.
|
||||
If `mode` is not specified, ``self.default_mode`` will be used.
|
||||
These values are usually initialized by passing the ``metric`` and
|
||||
``mode`` parameters to ``tune.run()``.
|
||||
|
||||
Args:
|
||||
metric: Key for trial info to order on. Defaults to
|
||||
``self.default_metric``.
|
||||
mode: One of [min, max]. Defaults to ``self.default_mode``.
|
||||
scope: One of [all, last, avg, last-5-avg, last-10-avg].
|
||||
If `scope=last`, only look at each trial's final step for
|
||||
`metric`, and compare across trials based on `mode=[min,max]`.
|
||||
If `scope=avg`, consider the simple average over all steps
|
||||
for `metric` and compare across trials based on
|
||||
`mode=[min,max]`. If `scope=last-5-avg` or `scope=last-10-avg`,
|
||||
consider the simple average over the last 5 or 10 steps for
|
||||
`metric` and compare across trials based on `mode=[min,max]`.
|
||||
If `scope=all`, find each trial's min/max score for `metric`
|
||||
based on `mode`, and compare trials based on `mode=[min,max]`.
|
||||
filter_nan_and_inf: If True (default), NaN or infinite
|
||||
values are disregarded and these trials are never selected as
|
||||
the best trial.
|
||||
|
||||
Returns:
|
||||
The best trial for the provided metric. If no trials contain the provided
|
||||
metric, or if the value for the metric is NaN for all trials,
|
||||
then returns None.
|
||||
"""
|
||||
if len(self.trials) == 1:
|
||||
return self.trials[0]
|
||||
|
||||
metric = self._validate_metric(metric)
|
||||
mode = self._validate_mode(mode)
|
||||
|
||||
if scope not in ["all", "last", "avg", "last-5-avg", "last-10-avg"]:
|
||||
raise ValueError(
|
||||
"ExperimentAnalysis: attempting to get best trial for "
|
||||
'metric {} for scope {} not in ["all", "last", "avg", '
|
||||
'"last-5-avg", "last-10-avg"]. '
|
||||
"If you didn't pass a `metric` parameter to `tune.run()`, "
|
||||
"you have to pass one when fetching the best trial.".format(
|
||||
metric, scope
|
||||
)
|
||||
)
|
||||
best_trial = None
|
||||
best_metric_score = None
|
||||
|
||||
for trial in self.trials:
|
||||
if metric not in trial.metric_analysis:
|
||||
continue
|
||||
|
||||
if scope in ["last", "avg", "last-5-avg", "last-10-avg"]:
|
||||
metric_score = trial.metric_analysis[metric][scope]
|
||||
else:
|
||||
metric_score = trial.metric_analysis[metric][mode]
|
||||
|
||||
if filter_nan_and_inf and is_nan_or_inf(metric_score):
|
||||
continue
|
||||
|
||||
if best_metric_score is None:
|
||||
best_metric_score = metric_score
|
||||
best_trial = trial
|
||||
continue
|
||||
|
||||
if (mode == "max") and (best_metric_score < metric_score):
|
||||
best_metric_score = metric_score
|
||||
best_trial = trial
|
||||
elif (mode == "min") and (best_metric_score > metric_score):
|
||||
best_metric_score = metric_score
|
||||
best_trial = trial
|
||||
|
||||
if not best_trial:
|
||||
logger.warning(
|
||||
"Could not find best trial. Did you pass the correct `metric` "
|
||||
"parameter?"
|
||||
)
|
||||
return best_trial
|
||||
|
||||
def get_best_config(
|
||||
self,
|
||||
metric: Optional[str] = None,
|
||||
mode: Optional[str] = None,
|
||||
scope: str = "last",
|
||||
) -> Optional[Dict]:
|
||||
"""Retrieve the best config corresponding to the trial.
|
||||
|
||||
Compares all trials' scores on `metric`.
|
||||
If ``metric`` is not specified, ``self.default_metric`` will be used.
|
||||
If `mode` is not specified, ``self.default_mode`` will be used.
|
||||
These values are usually initialized by passing the ``metric`` and
|
||||
``mode`` parameters to ``tune.run()``.
|
||||
|
||||
Args:
|
||||
metric: Key for trial info to order on. Defaults to
|
||||
``self.default_metric``.
|
||||
mode: One of [min, max]. Defaults to ``self.default_mode``.
|
||||
scope: One of [all, last, avg, last-5-avg, last-10-avg].
|
||||
If `scope=last`, only look at each trial's final step for
|
||||
`metric`, and compare across trials based on `mode=[min,max]`.
|
||||
If `scope=avg`, consider the simple average over all steps
|
||||
for `metric` and compare across trials based on
|
||||
`mode=[min,max]`. If `scope=last-5-avg` or `scope=last-10-avg`,
|
||||
consider the simple average over the last 5 or 10 steps for
|
||||
`metric` and compare across trials based on `mode=[min,max]`.
|
||||
If `scope=all`, find each trial's min/max score for `metric`
|
||||
based on `mode`, and compare trials based on `mode=[min,max]`.
|
||||
|
||||
Returns:
|
||||
The hyperparameter configuration of the best trial, or ``None`` if
|
||||
no best trial could be identified.
|
||||
"""
|
||||
best_trial = self.get_best_trial(metric, mode, scope)
|
||||
return best_trial.config if best_trial else None
|
||||
|
||||
def get_last_checkpoint(
|
||||
self,
|
||||
trial: Optional[Trial] = None,
|
||||
metric: str = "training_iteration",
|
||||
mode: str = "max",
|
||||
) -> Optional[Checkpoint]:
|
||||
"""Gets the last checkpoint of the provided trial,
|
||||
i.e., with the highest "training_iteration".
|
||||
|
||||
If no trial is specified, it loads the best trial according to the
|
||||
provided metric and mode (defaults to max. training iteration).
|
||||
|
||||
Args:
|
||||
trial: If None, load the best trial automatically.
|
||||
metric: If no trial is specified, use this metric to identify
|
||||
the best trial and load the last checkpoint from this trial.
|
||||
mode: If no trial is specified, use the metric and this mode
|
||||
to identify the best trial and load the last checkpoint from it.
|
||||
|
||||
Returns:
|
||||
Path for last checkpoint of trial
|
||||
"""
|
||||
trial = trial or self.get_best_trial(metric, mode)
|
||||
return self.get_best_checkpoint(trial, TRAINING_ITERATION, "max")
|
||||
|
||||
def _validate_metric(self, metric: str) -> str:
|
||||
if not metric and not self.default_metric:
|
||||
raise ValueError(
|
||||
"No `metric` has been passed and `default_metric` has "
|
||||
"not been set. Please specify the `metric` parameter."
|
||||
)
|
||||
return metric or self.default_metric
|
||||
|
||||
def _validate_mode(self, mode: str) -> str:
|
||||
if not mode and not self.default_mode:
|
||||
raise ValueError(
|
||||
"No `mode` has been passed and `default_mode` has "
|
||||
"not been set. Please specify the `mode` parameter."
|
||||
)
|
||||
if mode and mode not in ["min", "max"]:
|
||||
raise ValueError("If set, `mode` has to be one of [min, max]")
|
||||
return mode or self.default_mode
|
||||
|
||||
def _retrieve_rows(
|
||||
self, metric: Optional[str] = None, mode: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
assert mode is None or mode in ["max", "min"]
|
||||
assert not mode or metric
|
||||
rows = {}
|
||||
for path, df in self.trial_dataframes.items():
|
||||
if df.empty:
|
||||
continue
|
||||
if metric not in df:
|
||||
idx = -1
|
||||
elif mode == "max":
|
||||
idx = df[metric].idxmax()
|
||||
elif mode == "min":
|
||||
idx = df[metric].idxmin()
|
||||
else:
|
||||
idx = -1
|
||||
try:
|
||||
rows[path] = df.iloc[idx].to_dict()
|
||||
except TypeError:
|
||||
# idx is nan
|
||||
logger.warning(
|
||||
"Warning: Non-numerical value(s) encountered for {}".format(path)
|
||||
)
|
||||
|
||||
return rows
|
||||
|
||||
def __getstate__(self) -> Dict[str, Any]:
|
||||
"""Ensure that trials are marked as stubs when pickling,
|
||||
so that they can be loaded later without the trainable
|
||||
being registered.
|
||||
"""
|
||||
state = self.__dict__.copy()
|
||||
|
||||
def make_stub_if_needed(trial: Trial) -> Trial:
|
||||
if trial.stub:
|
||||
return trial
|
||||
trial_copy = Trial(trial.trainable_name, stub=True)
|
||||
trial_copy.__setstate__(trial.__getstate__())
|
||||
return trial_copy
|
||||
|
||||
state["trials"] = [make_stub_if_needed(t) for t in state["trials"]]
|
||||
return state
|
||||
Reference in New Issue
Block a user