wehub-resource-sync
560 lines
21 KiB
Python
560 lines
21 KiB
Python
import copy
|
|
import logging
|
|
from functools import partial
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
import numpy as np
|
|
|
|
# Use cloudpickle instead of pickle to make lambda funcs in HyperOpt pickleable
|
|
from ray import cloudpickle
|
|
from ray.tune.error import TuneError
|
|
from ray.tune.result import DEFAULT_METRIC
|
|
from ray.tune.search import (
|
|
UNDEFINED_METRIC_MODE,
|
|
UNDEFINED_SEARCH_SPACE,
|
|
UNRESOLVED_SEARCH_SPACE,
|
|
Searcher,
|
|
)
|
|
from ray.tune.search.sample import (
|
|
Categorical,
|
|
Domain,
|
|
Float,
|
|
Integer,
|
|
LogUniform,
|
|
Normal,
|
|
Quantized,
|
|
Uniform,
|
|
)
|
|
from ray.tune.search.variant_generator import assign_value, parse_spec_vars
|
|
from ray.tune.utils import flatten_dict
|
|
|
|
try:
|
|
hyperopt_logger = logging.getLogger("hyperopt")
|
|
hyperopt_logger.setLevel(logging.WARNING)
|
|
import hyperopt as hpo
|
|
from hyperopt.pyll import Apply
|
|
except ImportError:
|
|
hpo = None
|
|
Apply = None
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
HYPEROPT_UNDEFINED_DETAILS = (
|
|
" This issue can also come up with HyperOpt if your search space only "
|
|
"contains constant variables, which is not supported by HyperOpt. In that case, "
|
|
"don't pass any searcher or add sample variables to the search space."
|
|
)
|
|
|
|
|
|
class HyperOptSearch(Searcher):
|
|
"""A wrapper around HyperOpt to provide trial suggestions.
|
|
|
|
HyperOpt a Python library for serial and parallel optimization
|
|
over awkward search spaces, which may include real-valued, discrete,
|
|
and conditional dimensions. More info can be found at
|
|
http://hyperopt.github.io/hyperopt.
|
|
|
|
HyperOptSearch uses the Tree-structured Parzen Estimators algorithm,
|
|
though it can be trivially extended to support any algorithm HyperOpt
|
|
supports.
|
|
|
|
To use this search algorithm, you will need to install HyperOpt:
|
|
|
|
.. code-block:: bash
|
|
|
|
pip install -U hyperopt
|
|
|
|
|
|
Parameters:
|
|
space: HyperOpt configuration. Parameters will be sampled
|
|
from this configuration and will be used to override
|
|
parameters generated in the variant generation process.
|
|
metric: The training result objective value attribute. If None
|
|
but a mode was passed, the anonymous metric `_metric` will be used
|
|
per default.
|
|
mode: One of {min, max}. Determines whether objective is
|
|
minimizing or maximizing the metric attribute.
|
|
points_to_evaluate: Initial parameter suggestions to be run
|
|
first. This is for when you already have some good parameters
|
|
you want to run first to help the algorithm make better suggestions
|
|
for future parameters. Needs to be a list of dicts containing the
|
|
configurations.
|
|
n_initial_points: number of random evaluations of the
|
|
objective function before starting to approximate it with
|
|
tree parzen estimators. Defaults to 20.
|
|
random_state_seed: seed for reproducible
|
|
results. Defaults to None.
|
|
gamma: parameter governing the tree parzen
|
|
estimators suggestion algorithm. Defaults to 0.25.
|
|
|
|
Tune automatically converts search spaces to HyperOpt's format:
|
|
|
|
.. code-block:: python
|
|
|
|
config = {
|
|
'width': tune.uniform(0, 20),
|
|
'height': tune.uniform(-100, 100),
|
|
'activation': tune.choice(["relu", "tanh"])
|
|
}
|
|
|
|
current_best_params = [{
|
|
'width': 10,
|
|
'height': 0,
|
|
'activation': "relu",
|
|
}]
|
|
|
|
hyperopt_search = HyperOptSearch(
|
|
metric="mean_loss", mode="min",
|
|
points_to_evaluate=current_best_params)
|
|
|
|
tuner = tune.Tuner(
|
|
trainable,
|
|
tune_config=tune.TuneConfig(
|
|
search_alg=hyperopt_search
|
|
),
|
|
param_space=config
|
|
)
|
|
tuner.fit()
|
|
|
|
If you would like to pass the search space manually, the code would
|
|
look like this:
|
|
|
|
.. code-block:: python
|
|
|
|
space = {
|
|
'width': hp.uniform('width', 0, 20),
|
|
'height': hp.uniform('height', -100, 100),
|
|
'activation': hp.choice("activation", ["relu", "tanh"])
|
|
}
|
|
|
|
current_best_params = [{
|
|
'width': 10,
|
|
'height': 0,
|
|
'activation': "relu",
|
|
}]
|
|
|
|
hyperopt_search = HyperOptSearch(
|
|
space, metric="mean_loss", mode="min",
|
|
points_to_evaluate=current_best_params)
|
|
|
|
tuner = tune.Tuner(
|
|
trainable,
|
|
tune_config=tune.TuneConfig(
|
|
search_alg=hyperopt_search
|
|
),
|
|
)
|
|
tuner.fit()
|
|
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
space: Optional[Dict] = None,
|
|
metric: Optional[str] = None,
|
|
mode: Optional[str] = None,
|
|
points_to_evaluate: Optional[List[Dict]] = None,
|
|
n_initial_points: int = 20,
|
|
random_state_seed: Optional[int] = None,
|
|
gamma: float = 0.25,
|
|
):
|
|
assert (
|
|
hpo is not None
|
|
), "HyperOpt must be installed! Run `pip install hyperopt`."
|
|
if mode:
|
|
assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
|
|
super(HyperOptSearch, self).__init__(
|
|
metric=metric,
|
|
mode=mode,
|
|
)
|
|
# hyperopt internally minimizes, so "max" => -1
|
|
if mode == "max":
|
|
self.metric_op = -1.0
|
|
elif mode == "min":
|
|
self.metric_op = 1.0
|
|
|
|
if n_initial_points is None:
|
|
self.algo = hpo.tpe.suggest
|
|
else:
|
|
self.algo = partial(hpo.tpe.suggest, n_startup_jobs=n_initial_points)
|
|
if gamma is not None:
|
|
self.algo = partial(self.algo, gamma=gamma)
|
|
|
|
self._points_to_evaluate = copy.deepcopy(points_to_evaluate)
|
|
|
|
self._live_trial_mapping = {}
|
|
self.rstate = np.random.RandomState(random_state_seed)
|
|
|
|
self.domain = None
|
|
if isinstance(space, dict) and space:
|
|
resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
|
|
if domain_vars or grid_vars:
|
|
logger.warning(
|
|
UNRESOLVED_SEARCH_SPACE.format(par="space", cls=type(self))
|
|
)
|
|
space = self.convert_search_space(space)
|
|
self._space = space
|
|
self._setup_hyperopt()
|
|
|
|
def _setup_hyperopt(self) -> None:
|
|
from hyperopt.fmin import generate_trials_to_calculate
|
|
|
|
if not self._space:
|
|
raise RuntimeError(
|
|
UNDEFINED_SEARCH_SPACE.format(
|
|
cls=self.__class__.__name__, space="space"
|
|
)
|
|
+ HYPEROPT_UNDEFINED_DETAILS
|
|
)
|
|
|
|
if self._metric is None and self._mode:
|
|
# If only a mode was passed, use anonymous metric
|
|
self._metric = DEFAULT_METRIC
|
|
|
|
if self._points_to_evaluate is None:
|
|
self._hpopt_trials = hpo.Trials()
|
|
self._points_to_evaluate = 0
|
|
else:
|
|
assert isinstance(self._points_to_evaluate, (list, tuple))
|
|
|
|
for i in range(len(self._points_to_evaluate)):
|
|
config = self._points_to_evaluate[i]
|
|
self._convert_categories_to_indices(config)
|
|
# HyperOpt treats initial points as LIFO, reverse to get FIFO
|
|
self._points_to_evaluate = list(reversed(self._points_to_evaluate))
|
|
self._hpopt_trials = generate_trials_to_calculate(self._points_to_evaluate)
|
|
self._hpopt_trials.refresh()
|
|
self._points_to_evaluate = len(self._points_to_evaluate)
|
|
|
|
self.domain = hpo.Domain(lambda spc: spc, self._space)
|
|
|
|
def _convert_categories_to_indices(self, config) -> None:
|
|
"""Convert config parameters for categories into hyperopt-compatible
|
|
representations where instead the index of the category is expected."""
|
|
|
|
def _lookup(config_dict, space_dict, key):
|
|
if isinstance(config_dict[key], dict):
|
|
for k in config_dict[key]:
|
|
_lookup(config_dict[key], space_dict[key], k)
|
|
else:
|
|
if (
|
|
key in space_dict
|
|
and isinstance(space_dict[key], hpo.base.pyll.Apply)
|
|
and space_dict[key].name == "switch"
|
|
):
|
|
if len(space_dict[key].pos_args) > 0:
|
|
categories = [
|
|
a.obj
|
|
for a in space_dict[key].pos_args[1:]
|
|
if a.name == "literal"
|
|
]
|
|
try:
|
|
idx = categories.index(config_dict[key])
|
|
except ValueError as exc:
|
|
msg = (
|
|
f"Did not find category with value "
|
|
f"`{config_dict[key]}` in "
|
|
f"hyperopt parameter `{key}`. "
|
|
)
|
|
|
|
if isinstance(config_dict[key], int):
|
|
msg += (
|
|
"In previous versions, a numerical "
|
|
"index was expected for categorical "
|
|
"values of `points_to_evaluate`, "
|
|
"but in ray>=1.2.0, the categorical "
|
|
"value is expected to be directly "
|
|
"provided. "
|
|
)
|
|
|
|
msg += "Please make sure the specified category is valid."
|
|
raise ValueError(msg) from exc
|
|
config_dict[key] = idx
|
|
|
|
for k in config:
|
|
_lookup(config, self._space, k)
|
|
|
|
def set_search_properties(
|
|
self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
|
|
) -> bool:
|
|
if self.domain:
|
|
return False
|
|
space = self.convert_search_space(config)
|
|
self._space = space
|
|
|
|
if metric:
|
|
self._metric = metric
|
|
if mode:
|
|
self._mode = mode
|
|
|
|
self.metric_op = -1.0 if self._mode == "max" else 1.0
|
|
|
|
self._setup_hyperopt()
|
|
return True
|
|
|
|
def suggest(self, trial_id: str) -> Optional[Dict]:
|
|
if not self.domain:
|
|
raise RuntimeError(
|
|
UNDEFINED_SEARCH_SPACE.format(
|
|
cls=self.__class__.__name__, space="space"
|
|
)
|
|
+ HYPEROPT_UNDEFINED_DETAILS
|
|
)
|
|
if not self._metric or not self._mode:
|
|
raise RuntimeError(
|
|
UNDEFINED_METRIC_MODE.format(
|
|
cls=self.__class__.__name__, metric=self._metric, mode=self._mode
|
|
)
|
|
)
|
|
|
|
if self._points_to_evaluate > 0:
|
|
using_point_to_evaluate = True
|
|
new_trial = self._hpopt_trials.trials[self._points_to_evaluate - 1]
|
|
self._points_to_evaluate -= 1
|
|
else:
|
|
using_point_to_evaluate = False
|
|
new_ids = self._hpopt_trials.new_trial_ids(1)
|
|
self._hpopt_trials.refresh()
|
|
|
|
# Get new suggestion from Hyperopt
|
|
new_trials = self.algo(
|
|
new_ids,
|
|
self.domain,
|
|
self._hpopt_trials,
|
|
self.rstate.randint(2**31 - 1),
|
|
)
|
|
self._hpopt_trials.insert_trial_docs(new_trials)
|
|
self._hpopt_trials.refresh()
|
|
new_trial = new_trials[0]
|
|
self._live_trial_mapping[trial_id] = (new_trial["tid"], new_trial)
|
|
|
|
# Taken from HyperOpt.base.evaluate
|
|
config = hpo.base.spec_from_misc(new_trial["misc"])
|
|
|
|
# We have to flatten nested spaces here so parameter names match
|
|
config = flatten_dict(config, flatten_list=True)
|
|
|
|
ctrl = hpo.base.Ctrl(self._hpopt_trials, current_trial=new_trial)
|
|
memo = self.domain.memo_from_config(config)
|
|
hpo.utils.use_obj_for_literal_in_memo(
|
|
self.domain.expr, ctrl, hpo.base.Ctrl, memo
|
|
)
|
|
|
|
try:
|
|
suggested_config = hpo.pyll.rec_eval(
|
|
self.domain.expr,
|
|
memo=memo,
|
|
print_node_on_error=self.domain.rec_eval_print_node_on_error,
|
|
)
|
|
except (AssertionError, TypeError) as e:
|
|
if using_point_to_evaluate and (
|
|
isinstance(e, AssertionError) or "GarbageCollected" in str(e)
|
|
):
|
|
raise ValueError(
|
|
"HyperOpt encountered a GarbageCollected switch argument. "
|
|
"Usually this is caused by a config in "
|
|
"`points_to_evaluate` "
|
|
"missing a key present in `space`. Ensure that "
|
|
"`points_to_evaluate` contains "
|
|
"all non-constant keys from `space`.\n"
|
|
"Config from `points_to_evaluate`: "
|
|
f"{config}\n"
|
|
"HyperOpt search space: "
|
|
f"{self._space}"
|
|
) from e
|
|
raise e
|
|
return copy.deepcopy(suggested_config)
|
|
|
|
def on_trial_result(self, trial_id: str, result: Dict) -> None:
|
|
ho_trial = self._get_hyperopt_trial(trial_id)
|
|
if ho_trial is None:
|
|
return
|
|
now = hpo.utils.coarse_utcnow()
|
|
ho_trial["book_time"] = now
|
|
ho_trial["refresh_time"] = now
|
|
|
|
def on_trial_complete(
|
|
self, trial_id: str, result: Optional[Dict] = None, error: bool = False
|
|
) -> None:
|
|
"""Notification for the completion of trial.
|
|
|
|
The result is internally negated when interacting with HyperOpt
|
|
so that HyperOpt can "maximize" this value, as it minimizes on default.
|
|
"""
|
|
ho_trial = self._get_hyperopt_trial(trial_id)
|
|
if ho_trial is None:
|
|
return
|
|
ho_trial["refresh_time"] = hpo.utils.coarse_utcnow()
|
|
if error:
|
|
ho_trial["state"] = hpo.base.JOB_STATE_ERROR
|
|
ho_trial["misc"]["error"] = (str(TuneError), "Tune Error")
|
|
self._hpopt_trials.refresh()
|
|
elif result:
|
|
self._process_result(trial_id, result)
|
|
del self._live_trial_mapping[trial_id]
|
|
|
|
def _process_result(self, trial_id: str, result: Dict) -> None:
|
|
ho_trial = self._get_hyperopt_trial(trial_id)
|
|
if not ho_trial:
|
|
return
|
|
ho_trial["refresh_time"] = hpo.utils.coarse_utcnow()
|
|
|
|
ho_trial["state"] = hpo.base.JOB_STATE_DONE
|
|
hp_result = self._to_hyperopt_result(result)
|
|
ho_trial["result"] = hp_result
|
|
self._hpopt_trials.refresh()
|
|
|
|
def _to_hyperopt_result(self, result: Dict) -> Dict:
|
|
try:
|
|
return {"loss": self.metric_op * result[self.metric], "status": "ok"}
|
|
except KeyError as e:
|
|
raise RuntimeError(
|
|
f"Hyperopt expected to see the metric `{self.metric}` in the "
|
|
f"last result, but it was not found. To fix this, make "
|
|
f"sure your call to `tune.report` or your return value of "
|
|
f"your trainable class `step()` contains the above metric "
|
|
f"as a key."
|
|
) from e
|
|
|
|
def _get_hyperopt_trial(self, trial_id: str) -> Optional[Dict]:
|
|
if trial_id not in self._live_trial_mapping:
|
|
return
|
|
hyperopt_tid = self._live_trial_mapping[trial_id][0]
|
|
return [t for t in self._hpopt_trials.trials if t["tid"] == hyperopt_tid][0]
|
|
|
|
def get_state(self) -> Dict:
|
|
return {
|
|
"hyperopt_trials": self._hpopt_trials,
|
|
"rstate": self.rstate.get_state(),
|
|
}
|
|
|
|
def set_state(self, state: Dict) -> None:
|
|
self._hpopt_trials = state["hyperopt_trials"]
|
|
self.rstate.set_state(state["rstate"])
|
|
|
|
def save(self, checkpoint_path: str) -> None:
|
|
save_object = self.__dict__.copy()
|
|
save_object["__rstate"] = self.rstate.get_state()
|
|
with open(checkpoint_path, "wb") as f:
|
|
cloudpickle.dump(save_object, f)
|
|
|
|
def restore(self, checkpoint_path: str) -> None:
|
|
with open(checkpoint_path, "rb") as f:
|
|
save_object = cloudpickle.load(f)
|
|
|
|
if "__rstate" not in save_object:
|
|
# Backwards compatibility
|
|
self.set_state(save_object)
|
|
else:
|
|
self.rstate.set_state(save_object.pop("__rstate"))
|
|
self.__dict__.update(save_object)
|
|
|
|
@staticmethod
|
|
def convert_search_space(spec: Dict, prefix: str = "") -> Dict:
|
|
spec = copy.deepcopy(spec)
|
|
resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
|
|
|
|
if not domain_vars and not grid_vars:
|
|
return {}
|
|
|
|
if grid_vars:
|
|
raise ValueError(
|
|
"Grid search parameters cannot be automatically converted "
|
|
"to a HyperOpt search space."
|
|
)
|
|
|
|
def resolve_value(par: str, domain: Domain) -> Any:
|
|
quantize = None
|
|
|
|
sampler = domain.get_sampler()
|
|
if isinstance(sampler, Quantized):
|
|
quantize = sampler.q
|
|
sampler = sampler.sampler
|
|
|
|
if isinstance(domain, Float):
|
|
if isinstance(sampler, LogUniform):
|
|
if quantize:
|
|
return hpo.hp.qloguniform(
|
|
par, np.log(domain.lower), np.log(domain.upper), quantize
|
|
)
|
|
return hpo.hp.loguniform(
|
|
par, np.log(domain.lower), np.log(domain.upper)
|
|
)
|
|
elif isinstance(sampler, Uniform):
|
|
if quantize:
|
|
return hpo.hp.quniform(
|
|
par, domain.lower, domain.upper, quantize
|
|
)
|
|
return hpo.hp.uniform(par, domain.lower, domain.upper)
|
|
elif isinstance(sampler, Normal):
|
|
if quantize:
|
|
return hpo.hp.qnormal(par, sampler.mean, sampler.sd, quantize)
|
|
return hpo.hp.normal(par, sampler.mean, sampler.sd)
|
|
|
|
elif isinstance(domain, Integer):
|
|
if isinstance(sampler, LogUniform):
|
|
if quantize:
|
|
return hpo.base.pyll.scope.int(
|
|
hpo.hp.qloguniform(
|
|
par,
|
|
np.log(domain.lower),
|
|
np.log(domain.upper),
|
|
quantize,
|
|
)
|
|
)
|
|
return hpo.base.pyll.scope.int(
|
|
hpo.hp.qloguniform(
|
|
par, np.log(domain.lower), np.log(domain.upper - 1), 1.0
|
|
)
|
|
)
|
|
elif isinstance(sampler, Uniform):
|
|
if quantize:
|
|
return hpo.base.pyll.scope.int(
|
|
hpo.hp.quniform(
|
|
par, domain.lower, domain.upper - 1, quantize
|
|
)
|
|
)
|
|
return hpo.hp.uniformint(par, domain.lower, high=domain.upper - 1)
|
|
elif isinstance(domain, Categorical):
|
|
if isinstance(sampler, Uniform):
|
|
return hpo.hp.choice(
|
|
par,
|
|
[
|
|
(
|
|
HyperOptSearch.convert_search_space(
|
|
category, prefix=par
|
|
)
|
|
if isinstance(category, dict)
|
|
else (
|
|
HyperOptSearch.convert_search_space(
|
|
dict(enumerate(category)), prefix=f"{par}/{i}"
|
|
)
|
|
if isinstance(category, list)
|
|
and len(category) > 0
|
|
and isinstance(category[0], Domain)
|
|
else (
|
|
resolve_value(f"{par}/{i}", category)
|
|
if isinstance(category, Domain)
|
|
else category
|
|
)
|
|
)
|
|
)
|
|
for i, category in enumerate(domain.categories)
|
|
],
|
|
)
|
|
|
|
raise ValueError(
|
|
"HyperOpt does not support parameters of type "
|
|
"`{}` with samplers of type `{}`".format(
|
|
type(domain).__name__, type(domain.sampler).__name__
|
|
)
|
|
)
|
|
|
|
for path, domain in domain_vars:
|
|
par = "/".join([str(p) for p in ((prefix,) + path if prefix else path)])
|
|
value = resolve_value(par, domain)
|
|
assign_value(spec, path, value)
|
|
|
|
return spec
|