chore: import upstream snapshot with attribution

python/ray/tune/search/__init__.py

@@ -0,0 +1,149 @@
+from ray._common.utils import get_function_args
+from ray.tune.search.basic_variant import BasicVariantGenerator
+from ray.tune.search.concurrency_limiter import ConcurrencyLimiter
+from ray.tune.search.repeater import Repeater
+from ray.tune.search.search_algorithm import SearchAlgorithm
+from ray.tune.search.search_generator import SearchGenerator
+from ray.tune.search.searcher import Searcher
+from ray.tune.search.variant_generator import grid_search
+from ray.util import PublicAPI
+
+
+def _import_variant_generator():
+    return BasicVariantGenerator
+
+
+def _import_ax_search():
+    from ray.tune.search.ax.ax_search import AxSearch
+
+    return AxSearch
+
+
+def _import_hyperopt_search():
+    from ray.tune.search.hyperopt.hyperopt_search import HyperOptSearch
+
+    return HyperOptSearch
+
+
+def _import_bayesopt_search():
+    from ray.tune.search.bayesopt.bayesopt_search import BayesOptSearch
+
+    return BayesOptSearch
+
+
+def _import_bohb_search():
+    from ray.tune.search.bohb.bohb_search import TuneBOHB
+
+    return TuneBOHB
+
+
+def _import_nevergrad_search():
+    from ray.tune.search.nevergrad.nevergrad_search import NevergradSearch
+
+    return NevergradSearch
+
+
+def _import_optuna_search():
+    from ray.tune.search.optuna.optuna_search import OptunaSearch
+
+    return OptunaSearch
+
+
+def _import_zoopt_search():
+    from ray.tune.search.zoopt.zoopt_search import ZOOptSearch
+
+    return ZOOptSearch
+
+
+def _import_hebo_search():
+    from ray.tune.search.hebo.hebo_search import HEBOSearch
+
+    return HEBOSearch
+
+
+SEARCH_ALG_IMPORT = {
+    "variant_generator": _import_variant_generator,
+    "random": _import_variant_generator,
+    "ax": _import_ax_search,
+    "hyperopt": _import_hyperopt_search,
+    "bayesopt": _import_bayesopt_search,
+    "bohb": _import_bohb_search,
+    "nevergrad": _import_nevergrad_search,
+    "optuna": _import_optuna_search,
+    "zoopt": _import_zoopt_search,
+    "hebo": _import_hebo_search,
+}
+
+
+@PublicAPI(stability="beta")
+def create_searcher(
+    search_alg: str,
+    **kwargs,
+):
+    """Instantiate a search algorithm based on the given string.
+
+    This is useful for swapping between different search algorithms.
+
+    Args:
+        search_alg: The search algorithm to use.
+        **kwargs: Additional parameters (e.g. ``metric`` and ``mode``).
+            These keyword arguments will be passed to the initialization
+            function of the chosen class.
+    Returns:
+        ray.tune.search.Searcher: The search algorithm.
+    Example:
+        >>> from ray import tune # doctest: +SKIP
+        >>> search_alg = tune.create_searcher('ax') # doctest: +SKIP
+    """
+
+    search_alg = search_alg.lower()
+    if search_alg not in SEARCH_ALG_IMPORT:
+        raise ValueError(
+            f"The `search_alg` argument must be one of "
+            f"{list(SEARCH_ALG_IMPORT)}. "
+            f"Got: {search_alg}"
+        )
+
+    SearcherClass = SEARCH_ALG_IMPORT[search_alg]()
+
+    search_alg_args = get_function_args(SearcherClass)
+    trimmed_kwargs = {k: v for k, v in kwargs.items() if k in search_alg_args}
+
+    return SearcherClass(**trimmed_kwargs)
+
+
+UNRESOLVED_SEARCH_SPACE = str(
+    "You passed a `{par}` parameter to {cls} that contained unresolved search "
+    "space definitions. {cls} should however be instantiated with fully "
+    "configured search spaces only. To use Ray Tune's automatic search space "
+    "conversion, pass the space definition as part of the `param_space` argument "
+    "to `tune.Tuner()` instead."
+)
+
+UNDEFINED_SEARCH_SPACE = str(
+    "Trying to sample a configuration from {cls}, but no search "
+    "space has been defined. Either pass the `{space}` argument when "
+    "instantiating the search algorithm, or pass a `param_space` to "
+    "`tune.Tuner()`."
+)
+
+UNDEFINED_METRIC_MODE = str(
+    "Trying to sample a configuration from {cls}, but the `metric` "
+    "({metric}) or `mode` ({mode}) parameters have not been set. "
+    "Either pass these arguments when instantiating the search algorithm, "
+    "or pass them to `tune.TuneConfig()`."
+)
+
+
+__all__ = [
+    "SearchAlgorithm",
+    "Searcher",
+    "ConcurrencyLimiter",
+    "Repeater",
+    "BasicVariantGenerator",
+    "grid_search",
+    "SearchGenerator",
+    "UNRESOLVED_SEARCH_SPACE",
+    "UNDEFINED_SEARCH_SPACE",
+    "UNDEFINED_METRIC_MODE",
+]

python/ray/tune/search/_mock.py

@@ -0,0 +1,55 @@
+from typing import Dict, List, Optional
+
+from ray.tune.experiment import Trial
+from ray.tune.search import ConcurrencyLimiter, Searcher
+from ray.tune.search.search_generator import SearchGenerator
+
+
+class _MockSearcher(Searcher):
+    def __init__(self, **kwargs):
+        self.live_trials = {}
+        self.counter = {"result": 0, "complete": 0}
+        self.final_results = []
+        self.stall = False
+        self.results = []
+        super(_MockSearcher, self).__init__(**kwargs)
+
+    def suggest(self, trial_id: str):
+        if not self.stall:
+            self.live_trials[trial_id] = 1
+            return {"test_variable": 2}
+        return None
+
+    def on_trial_result(self, trial_id: str, result: Dict):
+        self.counter["result"] += 1
+        self.results += [result]
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ):
+        self.counter["complete"] += 1
+        if result:
+            self._process_result(result)
+        if trial_id in self.live_trials:
+            del self.live_trials[trial_id]
+
+    def _process_result(self, result: Dict):
+        self.final_results += [result]
+
+
+class _MockSuggestionAlgorithm(SearchGenerator):
+    def __init__(self, max_concurrent: Optional[int] = None, **kwargs):
+        self.searcher = _MockSearcher(**kwargs)
+        if max_concurrent:
+            self.searcher = ConcurrencyLimiter(
+                self.searcher, max_concurrent=max_concurrent
+            )
+        super(_MockSuggestionAlgorithm, self).__init__(self.searcher)
+
+    @property
+    def live_trials(self) -> List[Trial]:
+        return self.searcher.live_trials
+
+    @property
+    def results(self) -> List[Dict]:
+        return self.searcher.results

python/ray/tune/search/ax/__init__.py

@@ -0,0 +1,3 @@
+from ray.tune.search.ax.ax_search import AxSearch
+
+__all__ = ["AxSearch"]

python/ray/tune/search/ax/ax_search.py

@@ -0,0 +1,476 @@
+import copy
+import logging
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+
+from ray import cloudpickle
+from ray.tune.result import DEFAULT_METRIC
+from ray.tune.search import (
+    UNDEFINED_METRIC_MODE,
+    UNDEFINED_SEARCH_SPACE,
+    UNRESOLVED_SEARCH_SPACE,
+    Searcher,
+)
+from ray.tune.search.sample import (
+    Categorical,
+    Float,
+    Integer,
+    LogUniform,
+    Quantized,
+    Uniform,
+)
+from ray.tune.search.variant_generator import parse_spec_vars
+from ray.tune.utils.util import flatten_dict, unflatten_list_dict
+
+try:
+    import ax
+    from ax.service.ax_client import AxClient
+
+    try:
+        # Newer Ax versions
+        from ax.service.ax_client import ObjectiveProperties
+    except Exception:
+        ObjectiveProperties = None
+except ImportError:
+    ax = AxClient = ObjectiveProperties = None
+
+# This exception only exists in newer Ax releases for python 3.7
+try:
+    from ax.exceptions.core import DataRequiredError
+    from ax.exceptions.generation_strategy import MaxParallelismReachedException
+except ImportError:
+    MaxParallelismReachedException = DataRequiredError = Exception
+
+
+logger = logging.getLogger(__name__)
+
+
+class AxSearch(Searcher):
+    """Uses `Ax <https://ax.dev/>`_ to optimize hyperparameters.
+
+    Ax is a platform for understanding, managing, deploying, and
+    automating adaptive experiments. Ax provides an easy to use
+    interface with BoTorch, a flexible, modern library for Bayesian
+    optimization in PyTorch. More information can be found in https://ax.dev/.
+
+    To use this search algorithm, you must install Ax:
+
+    .. code-block:: bash
+
+        $ pip install ax-platform
+
+    Parameters:
+        space: Parameters in the experiment search space.
+            Required elements in the dictionaries are: "name" (name of
+            this parameter, string), "type" (type of the parameter: "range",
+            "fixed", or "choice", string), "bounds" for range parameters
+            (list of two values, lower bound first), "values" for choice
+            parameters (list of values), and "value" for fixed parameters
+            (single value).
+        metric: Name of the metric used as objective in this
+            experiment. This metric must be present in `raw_data` argument
+            to `log_data`. This metric must also be present in the dict
+            reported/returned by the Trainable. If None but a mode was passed,
+            the `ray.tune.result.DEFAULT_METRIC` will be used per default.
+        mode: One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute. Defaults to "max".
+        points_to_evaluate: Initial parameter suggestions to be run
+            first. This is for when you already have some good parameters
+            you want to run first to help the algorithm make better suggestions
+            for future parameters. Needs to be a list of dicts containing the
+            configurations.
+        parameter_constraints: Parameter constraints, such as
+            "x3 >= x4" or "x3 + x4 >= 2".
+        outcome_constraints: Outcome constraints of form
+            "metric_name >= bound", like "m1 <= 3."
+        ax_client: Optional AxClient instance. If this is set, do
+            not pass any values to these parameters: `space`, `metric`,
+            `parameter_constraints`, `outcome_constraints`.
+        **ax_kwargs: Passed to AxClient instance. Ignored if `AxClient` is not
+            None.
+
+    Tune automatically converts search spaces to Ax's format:
+
+    .. code-block:: python
+
+        from ray import tune
+        from ray.tune.search.ax import AxSearch
+
+        config = {
+            "x1": tune.uniform(0.0, 1.0),
+            "x2": tune.uniform(0.0, 1.0)
+        }
+
+        def easy_objective(config):
+            for i in range(100):
+                intermediate_result = config["x1"] + config["x2"] * i
+                tune.report({"score": intermediate_result})
+
+        ax_search = AxSearch()
+        tuner = tune.Tuner(
+            easy_objective,
+            tune_config=tune.TuneConfig(
+                search_alg=ax_search,
+                metric="score",
+                mode="max",
+            ),
+            param_space=config,
+        )
+        tuner.fit()
+
+    If you would like to pass the search space manually, the code would
+    look like this:
+
+    .. code-block:: python
+
+        from ray import tune
+        from ray.tune.search.ax import AxSearch
+
+        parameters = [
+            {"name": "x1", "type": "range", "bounds": [0.0, 1.0]},
+            {"name": "x2", "type": "range", "bounds": [0.0, 1.0]},
+        ]
+
+        def easy_objective(config):
+            for i in range(100):
+                intermediate_result = config["x1"] + config["x2"] * i
+                tune.report({"score": intermediate_result})
+
+        ax_search = AxSearch(space=parameters, metric="score", mode="max")
+        tuner = tune.Tuner(
+            easy_objective,
+            tune_config=tune.TuneConfig(
+                search_alg=ax_search,
+            ),
+        )
+        tuner.fit()
+
+    """
+
+    def __init__(
+        self,
+        space: Optional[Union[Dict, List[Dict]]] = None,
+        metric: Optional[str] = None,
+        mode: Optional[str] = None,
+        points_to_evaluate: Optional[List[Dict]] = None,
+        parameter_constraints: Optional[List] = None,
+        outcome_constraints: Optional[List] = None,
+        ax_client: Optional[AxClient] = None,
+        **ax_kwargs,
+    ):
+        assert (
+            ax is not None
+        ), """Ax must be installed!
+            You can install AxSearch with the command:
+            `pip install ax-platform`."""
+
+        if mode:
+            assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
+
+        super(AxSearch, self).__init__(
+            metric=metric,
+            mode=mode,
+        )
+
+        self._ax = ax_client
+        self._ax_kwargs = ax_kwargs or {}
+
+        if isinstance(space, dict) and space:
+            resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
+            if domain_vars or grid_vars:
+                logger.warning(
+                    UNRESOLVED_SEARCH_SPACE.format(par="space", cls=type(self))
+                )
+                space = self.convert_search_space(space)
+
+        self._space = space
+        self._parameter_constraints = parameter_constraints
+        self._outcome_constraints = outcome_constraints
+
+        self._points_to_evaluate = copy.deepcopy(points_to_evaluate)
+
+        self._parameters = []
+        self._live_trial_mapping = {}
+
+        if self._ax or self._space:
+            self._setup_experiment()
+
+    def _setup_experiment(self):
+        if self._metric is None and self._mode:
+            self._metric = DEFAULT_METRIC
+
+        if not self._ax:
+            self._ax = AxClient(**self._ax_kwargs)
+
+        # Detect whether the AxClient already has an experiment attached.
+        # ax 1.0+ uses AssertionError ("Experiment not set"); older Ax used ValueError.
+        try:
+            _ = self._ax.experiment
+            has_experiment = True
+        except AssertionError as e:
+            if "Experiment not set" not in str(e):
+                raise
+            has_experiment = False
+        except ValueError:
+            has_experiment = False
+
+        if has_experiment:
+            # User provided an already-initialized client; they must not also pass
+            # experiment-defining args to AxSearch.
+            if any(
+                [
+                    self._space,
+                    self._parameter_constraints,
+                    self._outcome_constraints,
+                    self._mode,
+                    self._metric,
+                ]
+            ):
+                raise ValueError(
+                    "If you create the Ax experiment yourself (by passing an AxClient "
+                    "with an experiment), do not pass values for these parameters to "
+                    "AxSearch: space, parameter_constraints, outcome_constraints, mode, metric."
+                )
+        else:
+            # AxSearch must create the experiment.
+            if not self._space:
+                raise ValueError(
+                    "You have to create an Ax experiment by calling "
+                    "`AxClient.create_experiment()`, or pass an Ax search space as "
+                    "`space` to AxSearch, or pass a `param_space` dict to `tune.Tuner()`."
+                )
+            if self._mode not in ["min", "max"]:
+                raise ValueError(
+                    "Please specify the `mode` argument when initializing the AxSearch "
+                    "object or pass it to `tune.TuneConfig()`."
+                )
+
+            minimize = self._mode != "max"
+
+            # New Ax API (objectives=...) vs old (objective_name/minimize)
+            try:
+                if ObjectiveProperties is None:
+                    raise TypeError(
+                        "ObjectiveProperties not available in this Ax version"
+                    )
+
+                self._ax.create_experiment(
+                    parameters=self._space,
+                    objectives={self._metric: ObjectiveProperties(minimize=minimize)},
+                    parameter_constraints=self._parameter_constraints,
+                    outcome_constraints=self._outcome_constraints,
+                )
+            except TypeError:
+                self._ax.create_experiment(
+                    parameters=self._space,
+                    objective_name=self._metric,
+                    parameter_constraints=self._parameter_constraints,
+                    outcome_constraints=self._outcome_constraints,
+                    minimize=minimize,
+                )
+
+        # Access experiment - should exist now (either created above or already existed)
+        try:
+            exp = self._ax.experiment
+        except AssertionError as e:
+            # Re-raise if the message is unexpected to avoid masking real Ax failures.
+            if "Experiment not set" not in str(e):
+                raise
+            raise RuntimeError(
+                "Failed to access Ax experiment after setup. "
+                "This may indicate an issue with the Ax client setup."
+            ) from e
+        except ValueError as e:
+            raise RuntimeError(
+                "Failed to access Ax experiment after setup. "
+                "This may indicate an issue with the Ax client setup."
+            ) from e
+
+        # Populate mode/metric from experiment (keep your existing logic here)
+        try:
+            obj = exp.optimization_config.objective
+            self._mode = "min" if obj.minimize else "max"
+            self._metric = obj.metric.name
+        except Exception:
+            objectives = exp.optimization_config.objective.objectives
+            first = objectives[0]
+            self._mode = "min" if first.minimize else "max"
+            self._metric = first.metric.name
+
+        self._parameters = list(exp.parameters)
+
+        if getattr(self._ax, "_enforce_sequential_optimization", False):
+            logger.warning(
+                "Detected sequential enforcement. Be sure to use a ConcurrencyLimiter."
+            )
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ):
+        if self._ax:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        self._setup_experiment()
+        return True
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        if not self._ax:
+            raise RuntimeError(
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"
+                )
+            )
+
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__, metric=self._metric, mode=self._mode
+                )
+            )
+
+        if self._points_to_evaluate:
+            config = self._points_to_evaluate.pop(0)
+            parameters, trial_index = self._ax.attach_trial(config)
+        else:
+            try:
+                parameters, trial_index = self._ax.get_next_trial()
+            except (MaxParallelismReachedException, DataRequiredError):
+                return None
+
+        self._live_trial_mapping[trial_id] = trial_index
+        try:
+            suggested_config = unflatten_list_dict(parameters)
+        except AssertionError:
+            # Fails to unflatten if keys are out of order, which only happens
+            # if search space includes a list with both constants and
+            # tunable hyperparameters:
+            # Ex: "a": [1, tune.uniform(2, 3), 4]
+            suggested_config = unflatten_list_dict(
+                {k: parameters[k] for k in sorted(parameters.keys())}
+            )
+        return suggested_config
+
+    def on_trial_complete(self, trial_id, result=None, error=False):
+        """Notification for the completion of trial.
+
+        Data of form key value dictionary of metric names and values.
+        """
+        if result:
+            self._process_result(trial_id, result)
+        self._live_trial_mapping.pop(trial_id)
+
+    def _process_result(self, trial_id, result):
+        ax_trial_index = self._live_trial_mapping[trial_id]
+        metrics_to_include = [self._metric] + [
+            oc.metric.name
+            for oc in self._ax.experiment.optimization_config.outcome_constraints
+        ]
+        metric_dict = {}
+        for key in metrics_to_include:
+            val = result[key]
+            if np.isnan(val) or np.isinf(val):
+                # Don't report trials with NaN metrics to Ax
+                self._ax.abandon_trial(
+                    trial_index=ax_trial_index,
+                    reason=f"nan/inf metrics reported by {trial_id}",
+                )
+                return
+            metric_dict[key] = (val, None)
+        self._ax.complete_trial(trial_index=ax_trial_index, raw_data=metric_dict)
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to an Ax search space."
+            )
+
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        def resolve_value(par, domain):
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning(
+                    "AxSearch does not support quantization. Dropped quantization."
+                )
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.lower, domain.upper],
+                        "value_type": "float",
+                        "log_scale": True,
+                    }
+                elif isinstance(sampler, Uniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.lower, domain.upper],
+                        "value_type": "float",
+                        "log_scale": False,
+                    }
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, LogUniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.lower, domain.upper - 1],
+                        "value_type": "int",
+                        "log_scale": True,
+                    }
+                elif isinstance(sampler, Uniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.lower, domain.upper - 1],
+                        "value_type": "int",
+                        "log_scale": False,
+                    }
+            elif isinstance(domain, Categorical):
+                if isinstance(sampler, Uniform):
+                    return {"name": par, "type": "choice", "values": domain.categories}
+
+            raise ValueError(
+                "AxSearch does not support parameters of type "
+                "`{}` with samplers of type `{}`".format(
+                    type(domain).__name__, type(domain.sampler).__name__
+                )
+            )
+
+        # Parameter name is e.g. "a/b/c" for nested dicts,
+        # "a/d/0", "a/d/1" for nested lists (using the index in the list)
+        fixed_values = [
+            {"name": "/".join(str(p) for p in path), "type": "fixed", "value": val}
+            for path, val in resolved_vars
+        ]
+        resolved_values = [
+            resolve_value("/".join(str(p) for p in path), domain)
+            for path, domain in domain_vars
+        ]
+
+        return fixed_values + resolved_values
+
+    def save(self, checkpoint_path: str):
+        save_object = self.__dict__
+        with open(checkpoint_path, "wb") as outputFile:
+            cloudpickle.dump(save_object, outputFile)
+
+    def restore(self, checkpoint_path: str):
+        with open(checkpoint_path, "rb") as inputFile:
+            save_object = cloudpickle.load(inputFile)
+        self.__dict__.update(save_object)

python/ray/tune/search/basic_variant.py

@@ -0,0 +1,424 @@
+import copy
+import itertools
+import os
+import uuid
+import warnings
+from pathlib import Path
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
+
+import numpy as np
+
+from ray.air._internal.usage import tag_searcher
+from ray.tune.error import TuneError
+from ray.tune.experiment.config_parser import _create_trial_from_spec, _make_parser
+from ray.tune.search.sample import _BackwardsCompatibleNumpyRng, np_random_generator
+from ray.tune.search.search_algorithm import SearchAlgorithm
+from ray.tune.search.variant_generator import (
+    _count_spec_samples,
+    _count_variants,
+    _flatten_resolved_vars,
+    _get_preset_variants,
+    format_vars,
+    generate_variants,
+)
+from ray.tune.utils.util import _atomic_save, _load_newest_checkpoint
+from ray.util import PublicAPI
+
+if TYPE_CHECKING:
+    from ray.tune.experiment import Experiment
+
+SERIALIZATION_THRESHOLD = 1e6
+
+
+class _VariantIterator:
+    """Iterates over generated variants from the search space.
+
+    This object also toggles between lazy evaluation and
+    eager evaluation of samples. If lazy evaluation is enabled,
+    this object cannot be serialized.
+    """
+
+    def __init__(self, iterable, lazy_eval=False):
+        self.lazy_eval = lazy_eval
+        self.iterable = iterable
+        self._has_next = True
+        if lazy_eval:
+            self._load_value()
+        else:
+            self.iterable = list(iterable)
+            self._has_next = bool(self.iterable)
+
+    def _load_value(self):
+        try:
+            self.next_value = next(self.iterable)
+        except StopIteration:
+            self._has_next = False
+
+    def has_next(self):
+        return self._has_next
+
+    def __next__(self):
+        if self.lazy_eval:
+            current_value = self.next_value
+            self._load_value()
+            return current_value
+        current_value = self.iterable.pop(0)
+        self._has_next = bool(self.iterable)
+        return current_value
+
+
+class _TrialIterator:
+    """Generates trials from the spec.
+
+    Args:
+        uuid_prefix: Used in creating the trial name.
+        num_samples: Number of samples from distribution
+             (same as tune.TuneConfig).
+        unresolved_spec: Experiment specification
+            that might have unresolved distributions.
+        constant_grid_search: Should random variables be sampled
+            first before iterating over grid variants (True) or not (False).
+        points_to_evaluate: Configurations that will be tried out without sampling.
+        lazy_eval: Whether variants should be generated
+            lazily or eagerly. This is toggled depending
+            on the size of the grid search.
+        start: index at which to start counting trials.
+        random_state: Seed or numpy random generator to use for reproducible results.
+            If None (default), will use the global numpy random generator
+            (``np.random``). Please note that full reproducibility cannot
+            be guaranteed in a distributed environment.
+    """
+
+    def __init__(
+        self,
+        uuid_prefix: str,
+        num_samples: int,
+        unresolved_spec: dict,
+        constant_grid_search: bool = False,
+        points_to_evaluate: Optional[List] = None,
+        lazy_eval: bool = False,
+        start: int = 0,
+        random_state: Optional[
+            Union[int, "np_random_generator", np.random.RandomState]
+        ] = None,
+    ):
+        self.parser = _make_parser()
+        self.num_samples = num_samples
+        self.uuid_prefix = uuid_prefix
+        self.num_samples_left = num_samples
+        self.unresolved_spec = unresolved_spec
+        self.constant_grid_search = constant_grid_search
+        self.points_to_evaluate = points_to_evaluate or []
+        self.num_points_to_evaluate = len(self.points_to_evaluate)
+        self.counter = start
+        self.lazy_eval = lazy_eval
+        self.variants = None
+        self.random_state = random_state
+
+    def create_trial(self, resolved_vars, spec):
+        trial_id = self.uuid_prefix + ("%05d" % self.counter)
+        experiment_tag = str(self.counter)
+        # Always append resolved vars to experiment tag?
+        if resolved_vars:
+            experiment_tag += "_{}".format(format_vars(resolved_vars))
+        self.counter += 1
+        return _create_trial_from_spec(
+            spec,
+            self.parser,
+            evaluated_params=_flatten_resolved_vars(resolved_vars),
+            trial_id=trial_id,
+            experiment_tag=experiment_tag,
+        )
+
+    def __next__(self):
+        """Generates Trial objects with the variant generation process.
+
+        Uses a fixed point iteration to resolve variants. All trials
+        should be able to be generated at once.
+
+        See also: `ray.tune.search.variant_generator`.
+
+        Returns:
+            Trial object
+        """
+
+        if "run" not in self.unresolved_spec:
+            raise TuneError("Must specify `run` in {}".format(self.unresolved_spec))
+
+        if self.variants and self.variants.has_next():
+            # This block will be skipped upon instantiation.
+            # `variants` will be set later after the first loop.
+            resolved_vars, spec = next(self.variants)
+            return self.create_trial(resolved_vars, spec)
+
+        if self.points_to_evaluate:
+            config = self.points_to_evaluate.pop(0)
+            self.num_samples_left -= 1
+            self.variants = _VariantIterator(
+                _get_preset_variants(
+                    self.unresolved_spec,
+                    config,
+                    constant_grid_search=self.constant_grid_search,
+                    random_state=self.random_state,
+                ),
+                lazy_eval=self.lazy_eval,
+            )
+            resolved_vars, spec = next(self.variants)
+            return self.create_trial(resolved_vars, spec)
+        elif self.num_samples_left > 0:
+            self.variants = _VariantIterator(
+                generate_variants(
+                    self.unresolved_spec,
+                    constant_grid_search=self.constant_grid_search,
+                    random_state=self.random_state,
+                ),
+                lazy_eval=self.lazy_eval,
+            )
+            self.num_samples_left -= 1
+            resolved_vars, spec = next(self.variants)
+            return self.create_trial(resolved_vars, spec)
+        else:
+            raise StopIteration
+
+    def __iter__(self):
+        return self
+
+
+@PublicAPI
+class BasicVariantGenerator(SearchAlgorithm):
+    """Uses Tune's variant generation for resolving variables.
+
+    This is the default search algorithm used if no other search algorithm
+    is specified.
+
+
+    Args:
+        points_to_evaluate: Initial parameter suggestions to be run
+            first. This is for when you already have some good parameters
+            you want to run first to help the algorithm make better suggestions
+            for future parameters. Needs to be a list of dicts containing the
+            configurations.
+        max_concurrent: Maximum number of concurrently running trials.
+            If 0 (default), no maximum is enforced.
+        constant_grid_search: If this is set to ``True``, Ray Tune will
+            *first* try to sample random values and keep them constant over
+            grid search parameters. If this is set to ``False`` (default),
+            Ray Tune will sample new random parameters in each grid search
+            condition.
+        random_state:
+            Seed or numpy random generator to use for reproducible results.
+            If None (default), will use the global numpy random generator
+            (``np.random``). Please note that full reproducibility cannot
+            be guaranteed in a distributed environment.
+
+
+    Example:
+
+    .. code-block:: python
+
+        from ray import tune
+
+        # This will automatically use the `BasicVariantGenerator`
+        tuner = tune.Tuner(
+            lambda config: config["a"] + config["b"],
+            tune_config=tune.TuneConfig(
+                num_samples=4
+            ),
+            param_space={
+                "a": tune.grid_search([1, 2]),
+                "b": tune.randint(0, 3)
+            },
+        )
+        tuner.fit()
+
+    In the example above, 8 trials will be generated: For each sample
+    (``4``), each of the grid search variants for ``a`` will be sampled
+    once. The ``b`` parameter will be sampled randomly.
+
+    The generator accepts a pre-set list of points that should be evaluated.
+    The points will replace the first samples of each experiment passed to
+    the ``BasicVariantGenerator``.
+
+    Each point will replace one sample of the specified ``num_samples``. If
+    grid search variables are overwritten with the values specified in the
+    presets, the number of samples will thus be reduced.
+
+    Example:
+
+    .. code-block:: python
+
+        from ray import tune
+        from ray.tune.search.basic_variant import BasicVariantGenerator
+
+        tuner = tune.Tuner(
+            lambda config: config["a"] + config["b"],
+            tune_config=tune.TuneConfig(
+                search_alg=BasicVariantGenerator(points_to_evaluate=[
+                    {"a": 2, "b": 2},
+                    {"a": 1},
+                    {"b": 2}
+                ]),
+                num_samples=4
+            ),
+            param_space={
+                "a": tune.grid_search([1, 2]),
+                "b": tune.randint(0, 3)
+            },
+        )
+        tuner.fit()
+
+    The example above will produce six trials via four samples:
+
+    - The first sample will produce one trial with ``a=2`` and ``b=2``.
+    - The second sample will produce one trial with ``a=1`` and ``b`` sampled
+      randomly
+    - The third sample will produce two trials, one for each grid search
+      value of ``a``. It will be ``b=2`` for both of these trials.
+    - The fourth sample will produce two trials, one for each grid search
+      value of ``a``. ``b`` will be sampled randomly and independently for
+      both of these trials.
+
+    """
+
+    CKPT_FILE_TMPL = "basic-variant-state-{}.json"
+
+    def __init__(
+        self,
+        points_to_evaluate: Optional[List[Dict]] = None,
+        max_concurrent: int = 0,
+        constant_grid_search: bool = False,
+        random_state: Optional[
+            Union[int, "np_random_generator", np.random.RandomState]
+        ] = None,
+    ):
+        tag_searcher(self)
+        self._trial_generator = []
+        self._iterators = []
+        self._trial_iter = None
+        self._finished = False
+        self._random_state = _BackwardsCompatibleNumpyRng(random_state)
+
+        self._points_to_evaluate = points_to_evaluate or []
+
+        # Unique prefix for all trials generated, e.g., trial ids start as
+        # 2f1e_00001, 2f1ef_00002, 2f1ef_0003, etc. Overridable for testing.
+        force_test_uuid = os.environ.get("_TEST_TUNE_TRIAL_UUID")
+        if force_test_uuid:
+            self._uuid_prefix = force_test_uuid + "_"
+        else:
+            self._uuid_prefix = str(uuid.uuid1().hex)[:5] + "_"
+
+        self._total_samples = 0
+        self.max_concurrent = max_concurrent
+        self._constant_grid_search = constant_grid_search
+        self._live_trials = set()
+
+    @property
+    def total_samples(self):
+        return self._total_samples
+
+    def add_configurations(
+        self, experiments: Union["Experiment", List["Experiment"], Dict[str, Dict]]
+    ):
+        """Chains generator given experiment specifications.
+
+        Arguments:
+            experiments: Experiments to run.
+        """
+        from ray.tune.experiment import _convert_to_experiment_list
+
+        experiment_list = _convert_to_experiment_list(experiments)
+
+        for experiment in experiment_list:
+            grid_vals = _count_spec_samples(experiment.spec, num_samples=1)
+            lazy_eval = grid_vals > SERIALIZATION_THRESHOLD
+            if lazy_eval:
+                warnings.warn(
+                    f"The number of pre-generated samples ({grid_vals}) "
+                    "exceeds the serialization threshold "
+                    f"({int(SERIALIZATION_THRESHOLD)}). Resume ability is "
+                    "disabled. To fix this, reduce the number of "
+                    "dimensions/size of the provided grid search."
+                )
+
+            previous_samples = self._total_samples
+            points_to_evaluate = copy.deepcopy(self._points_to_evaluate)
+            self._total_samples += _count_variants(experiment.spec, points_to_evaluate)
+            iterator = _TrialIterator(
+                uuid_prefix=self._uuid_prefix,
+                num_samples=experiment.spec.get("num_samples", 1),
+                unresolved_spec=experiment.spec,
+                constant_grid_search=self._constant_grid_search,
+                points_to_evaluate=points_to_evaluate,
+                lazy_eval=lazy_eval,
+                start=previous_samples,
+                random_state=self._random_state,
+            )
+            self._iterators.append(iterator)
+            self._trial_generator = itertools.chain(self._trial_generator, iterator)
+
+    def next_trial(self):
+        """Provides one Trial object to be queued into the TrialRunner.
+
+        Returns:
+            Trial: Returns a single trial.
+        """
+        if self.is_finished():
+            return None
+        if self.max_concurrent > 0 and len(self._live_trials) >= self.max_concurrent:
+            return None
+        if not self._trial_iter:
+            self._trial_iter = iter(self._trial_generator)
+        try:
+            trial = next(self._trial_iter)
+            self._live_trials.add(trial.trial_id)
+            return trial
+        except StopIteration:
+            self._trial_generator = []
+            self._trial_iter = None
+            self.set_finished()
+            return None
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ):
+        if trial_id in self._live_trials:
+            self._live_trials.remove(trial_id)
+
+    def get_state(self):
+        if any(iterator.lazy_eval for iterator in self._iterators):
+            return False
+        state = self.__dict__.copy()
+        del state["_trial_generator"]
+        del state["_trial_iter"]
+        return state
+
+    def set_state(self, state):
+        self.__dict__.update(state)
+        self._trial_iter = None
+        self._trial_generator = []
+        for iterator in self._iterators:
+            self._trial_generator = itertools.chain(self._trial_generator, iterator)
+
+    def save_to_dir(self, dirpath, session_str):
+        if any(iterator.lazy_eval for iterator in self._iterators):
+            return False
+        state_dict = self.get_state()
+        file_name = self.CKPT_FILE_TMPL.format(session_str)
+        _atomic_save(
+            state=state_dict,
+            checkpoint_dir=dirpath,
+            file_name=file_name,
+            tmp_file_name=f"tmp-{file_name}",
+        )
+
+    def has_checkpoint(self, dirpath: str):
+        """Whether a checkpoint file exists within dirpath."""
+        return any(Path(dirpath).glob(self.CKPT_FILE_TMPL.format("*")))
+
+    def restore_from_dir(self, dirpath: str):
+        """Restores self + searcher + search wrappers from dirpath."""
+        state_dict = _load_newest_checkpoint(dirpath, self.CKPT_FILE_TMPL.format("*"))
+        if not state_dict:
+            raise RuntimeError("Unable to find checkpoint in {}.".format(dirpath))
+        self.set_state(state_dict)

python/ray/tune/search/bayesopt/__init__.py

@@ -0,0 +1,3 @@
+from ray.tune.search.bayesopt.bayesopt_search import BayesOptSearch
+
+__all__ = ["BayesOptSearch"]

python/ray/tune/search/bayesopt/bayesopt_search.py

@@ -0,0 +1,493 @@
+import json
+import logging
+import pickle
+from collections import defaultdict
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+
+from ray.tune.result import DEFAULT_METRIC
+from ray.tune.search import (
+    UNDEFINED_METRIC_MODE,
+    UNDEFINED_SEARCH_SPACE,
+    UNRESOLVED_SEARCH_SPACE,
+    Searcher,
+)
+from ray.tune.search.sample import Domain, Float, Quantized, Uniform
+from ray.tune.search.variant_generator import parse_spec_vars
+from ray.tune.utils import flatten_dict
+from ray.tune.utils.util import is_nan_or_inf, unflatten_dict
+
+try:  # Python 3 only -- needed for lint test.
+    import bayes_opt as byo
+except ImportError:
+    byo = None
+
+
+if TYPE_CHECKING:
+    from ray.tune import ExperimentAnalysis
+
+logger = logging.getLogger(__name__)
+
+
+def _dict_hash(config, precision):
+    flatconfig = flatten_dict(config)
+    for param, value in flatconfig.items():
+        if isinstance(value, float):
+            flatconfig[param] = "{:.{digits}f}".format(value, digits=precision)
+
+    hashed = json.dumps(flatconfig, sort_keys=True, default=str)
+    return hashed
+
+
+class BayesOptSearch(Searcher):
+    """Uses bayesian-optimization/BayesianOptimization to optimize hyperparameters.
+
+    bayesian-optimization/BayesianOptimization is a library for Bayesian Optimization. More
+    info can be found here: https://github.com/bayesian-optimization/BayesianOptimization.
+
+    This searcher will automatically filter out any NaN, inf or -inf
+    results.
+
+    You will need to install bayesian-optimization/BayesianOptimization via the following:
+
+    .. code-block:: bash
+
+        pip install bayesian-optimization==1.4.3
+
+    Initializing this search algorithm with a ``space`` requires that it's
+    in the ``BayesianOptimization`` search space format. Otherwise, you
+    should instead pass in a Tune search space into ``Tuner(param_space=...)``,
+    and the search space will be automatically converted for you.
+
+    See this ``BayesianOptimization`` example notebook
+    <https://github.com/bayesian-optimization/BayesianOptimization/blob/33b99ec0a4fc51239e1a2fca3eaa37ad6debac5d/examples/advanced-tour.ipynb>`_
+    for an example.
+
+    Args:
+        space: Continuous search space. Parameters will be sampled from
+            this space which will be used to run trials.
+        metric: The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
+        mode: One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute.
+        points_to_evaluate: Initial parameter suggestions to be run
+            first. This is for when you already have some good parameters
+            you want to run first to help the algorithm make better suggestions
+            for future parameters. Needs to be a list of dicts containing the
+            configurations.
+        utility_kwargs: Parameters to define the utility function.
+            The default value is a dictionary with three keys:
+            - kind: ucb (Upper Confidence Bound)
+            - kappa: 2.576
+            - xi: 0.0
+        random_state: Used to initialize BayesOpt.
+        random_search_steps: Number of initial random searches.
+            This is necessary to avoid initial local overfitting
+            of the Bayesian process.
+        verbose: Sets verbosity level for BayesOpt packages.
+        patience: Number of times a configuration may be repeatedly suggested
+            before the search terminates early. The Bayesian optimizer can
+            converge and keep proposing the same point; when a configuration is
+            suggested more than ``patience`` times, ``suggest`` returns
+            ``Searcher.FINISHED`` and no further trials are started (so a run
+            may end before ``num_samples``). Set ``patience=1`` to stop as soon
+            as a configuration first repeats. Defaults to 5.
+        skip_duplicate: If True (default), configurations that have already been
+            suggested are skipped instead of re-evaluated. Set to False to allow
+            duplicate suggestions (useful for noisy objectives, or to keep
+            running until ``num_samples`` is reached).
+        analysis: Optionally, the previous analysis to integrate.
+        repeat_float_precision: Decimal precision used when hashing float
+            values in a config to detect duplicate suggestions. Higher
+            values make the duplicate check stricter, so fewer distinct
+            configurations are treated as repeats and skipped. Defaults to 5.
+
+    Tune automatically converts search spaces to BayesOptSearch's format:
+
+    .. code-block:: python
+
+        from ray import tune
+        from ray.tune.search.bayesopt import BayesOptSearch
+
+        config = {
+            "width": tune.uniform(0, 20),
+            "height": tune.uniform(-100, 100)
+        }
+
+        bayesopt = BayesOptSearch(metric="mean_loss", mode="min")
+        tuner = tune.Tuner(
+            my_func,
+            tune_config=tune.TuneConfig(
+                search_alg=baysopt,
+            ),
+            param_space=config,
+        )
+        tuner.fit()
+
+    If you would like to pass the search space manually, the code would
+    look like this:
+
+    .. code-block:: python
+
+        from ray import tune
+        from ray.tune.search.bayesopt import BayesOptSearch
+
+        space = {
+            'width': (0, 20),
+            'height': (-100, 100),
+        }
+        bayesopt = BayesOptSearch(space, metric="mean_loss", mode="min")
+        tuner = tune.Tuner(
+            my_func,
+            tune_config=tune.TuneConfig(
+                search_alg=bayesopt,
+            ),
+        )
+        tuner.fit()
+
+    """
+
+    # bayes_opt.BayesianOptimization: Optimization object
+    optimizer = None
+
+    def __init__(
+        self,
+        space: Optional[Dict] = None,
+        metric: Optional[str] = None,
+        mode: Optional[str] = None,
+        points_to_evaluate: Optional[List[Dict]] = None,
+        utility_kwargs: Optional[Dict] = None,
+        random_state: int = 42,
+        random_search_steps: int = 10,
+        verbose: int = 0,
+        patience: int = 5,
+        skip_duplicate: bool = True,
+        analysis: Optional["ExperimentAnalysis"] = None,
+        repeat_float_precision: int = 5,
+    ):
+        assert byo is not None, (
+            "BayesOpt must be installed!. You can install BayesOpt with"
+            " the command: `pip install bayesian-optimization==1.4.3`."
+        )
+        if mode:
+            assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
+        self._config_counter = defaultdict(int)
+        self._patience = patience
+        if isinstance(repeat_float_precision, bool) or not isinstance(
+            repeat_float_precision, int
+        ):
+            raise TypeError("repeat_float_precision must be an integer.")
+        if repeat_float_precision < 0:
+            raise ValueError("repeat_float_precision must be non-negative.")
+        # int: Precision at which to hash float values when checking for
+        # duplicate suggestions.
+        self.repeat_float_precision = repeat_float_precision
+        if self._patience <= 0:
+            raise ValueError("patience must be set to a value greater than 0!")
+        self._skip_duplicate = skip_duplicate
+        # One-time warnings so a converged GP does not fail silently.
+        self._logged_duplicate_warning = False
+        self._logged_convergence_stop_warning = False
+        super(BayesOptSearch, self).__init__(
+            metric=metric,
+            mode=mode,
+        )
+
+        if utility_kwargs is None:
+            # The defaults arguments are the same
+            # as in the package BayesianOptimization
+            utility_kwargs = dict(
+                kind="ucb",
+                kappa=2.576,
+                xi=0.0,
+            )
+
+        if mode == "max":
+            self._metric_op = 1.0
+        elif mode == "min":
+            self._metric_op = -1.0
+
+        self._points_to_evaluate = points_to_evaluate
+
+        self._live_trial_mapping = {}
+        self._buffered_trial_results = []
+        self.random_search_trials = random_search_steps
+        self._total_random_search_trials = 0
+
+        self.utility = byo.UtilityFunction(**utility_kwargs)
+
+        self._analysis = analysis
+
+        if isinstance(space, dict) and space:
+            resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
+            if domain_vars or grid_vars:
+                logger.warning(
+                    UNRESOLVED_SEARCH_SPACE.format(par="space", cls=type(self))
+                )
+                space = self.convert_search_space(space, join=True)
+
+        self._space = space
+        self._verbose = verbose
+        self._random_state = random_state
+
+        self.optimizer = None
+        if space:
+            self._setup_optimizer()
+
+    def _setup_optimizer(self):
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
+        self.optimizer = byo.BayesianOptimization(
+            f=None,
+            pbounds=self._space,
+            verbose=self._verbose,
+            random_state=self._random_state,
+        )
+
+        # Registering the provided analysis, if given
+        if self._analysis is not None:
+            self.register_analysis(self._analysis)
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        if self.optimizer:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        if self._mode == "max":
+            self._metric_op = 1.0
+        elif self._mode == "min":
+            self._metric_op = -1.0
+
+        self._setup_optimizer()
+        return True
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        """Return new point to be explored by black box function.
+
+        Args:
+            trial_id: Id of the trial.
+                This is a short alphanumerical string.
+
+        Returns:
+            Either a dictionary describing the new point to explore or
+            None, when no new point is to be explored for the time being.
+        """
+        if not self.optimizer:
+            raise RuntimeError(
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"
+                )
+            )
+
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__, metric=self._metric, mode=self._mode
+                )
+            )
+
+        if self._points_to_evaluate:
+            config = self._points_to_evaluate.pop(0)
+        else:
+            # We compute the new point to explore
+            config = self.optimizer.suggest(self.utility)
+
+        config_hash = _dict_hash(config, self.repeat_float_precision)
+        # Check if already computed
+        already_seen = config_hash in self._config_counter
+        self._config_counter[config_hash] += 1
+        top_repeats = max(self._config_counter.values())
+
+        # If patience is set and we've repeated a trial numerous times,
+        # we terminate the experiment.
+        if self._patience is not None and top_repeats > self._patience:
+            if not self._logged_convergence_stop_warning:
+                logger.warning(
+                    "BayesOptSearch is stopping early: a configuration was "
+                    "suggested more than `patience` (%d) times, which usually "
+                    "means the search has converged. No further trials will be "
+                    "suggested. To run more trials, increase `patience`, set "
+                    "`skip_duplicate=False`, or widen the search space."
+                    % self._patience
+                )
+                self._logged_convergence_stop_warning = True
+            return Searcher.FINISHED
+        # If we have seen a value before, we'll skip it.
+        if already_seen and self._skip_duplicate:
+            if not self._logged_duplicate_warning:
+                logger.warning(
+                    "BayesOptSearch is re-suggesting already-evaluated "
+                    "configurations (the Gaussian Process has likely "
+                    "converged). Duplicates are being skipped; if a "
+                    "configuration repeats more than `patience` (%d) times the "
+                    "search will stop early. Set `skip_duplicate=False`, "
+                    "increase `patience`, or widen the search space to keep "
+                    "exploring." % self._patience
+                )
+                self._logged_duplicate_warning = True
+            logger.info("Skipping duplicated config: {}.".format(config))
+            return None
+
+        # If we are still in the random search part and we are waiting for
+        # trials to complete
+        if len(self._buffered_trial_results) < self.random_search_trials:
+            # We check if we have already maxed out the number of requested
+            # random search trials
+            if self._total_random_search_trials == self.random_search_trials:
+                # If so we stop the suggestion and return None
+                return None
+            # Otherwise we increase the total number of rndom search trials
+            if config:
+                self._total_random_search_trials += 1
+
+        # Save the new trial to the trial mapping
+        self._live_trial_mapping[trial_id] = config
+
+        # Return a deep copy of the mapping
+        return unflatten_dict(config)
+
+    def register_analysis(self, analysis: "ExperimentAnalysis"):
+        """Integrate the given analysis into the gaussian process.
+
+        Args:
+            analysis: Optionally, the previous analysis
+                to integrate.
+        """
+        for (_, report), params in zip(
+            analysis.dataframe(metric=self._metric, mode=self._mode).iterrows(),
+            analysis.get_all_configs().values(),
+        ):
+            # We add the obtained results to the
+            # gaussian process optimizer
+            self._register_result(params, report)
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ) -> None:
+        """Notification for the completion of trial.
+
+        Args:
+            trial_id: Id of the trial.
+                This is a short alphanumerical string.
+            result: Dictionary of result.
+                May be none when some error occurs.
+            error: Boolean representing a previous error state.
+                The result should be None when error is True.
+        """
+        # We try to get the parameters used for this trial
+        params = self._live_trial_mapping.pop(trial_id, None)
+
+        # The results may be None if some exception is raised during the trial.
+        # Also, if the parameters are None (were already processed)
+        # we interrupt the following procedure.
+        # Additionally, if somehow the error is True but
+        # the remaining values are not we also block the method
+        if result is None or params is None or error:
+            return
+
+        # If we don't have to execute some random search steps
+        if len(self._buffered_trial_results) >= self.random_search_trials:
+            #  we simply register the obtained result
+            self._register_result(params, result)
+            return
+
+        # We store the results into a temporary cache
+        self._buffered_trial_results.append((params, result))
+
+        # If the random search finished,
+        # we update the BO with all the computer points.
+        if len(self._buffered_trial_results) == self.random_search_trials:
+            for params, result in self._buffered_trial_results:
+                self._register_result(params, result)
+
+    def _register_result(self, params: Tuple[str], result: Dict):
+        """Register given tuple of params and results."""
+        if is_nan_or_inf(result[self.metric]):
+            return
+        self.optimizer.register(params, self._metric_op * result[self.metric])
+
+    def get_state(self) -> Dict[str, Any]:
+        state = self.__dict__.copy()
+        return state
+
+    def set_state(self, state: Dict[str, Any]):
+        self.__dict__.update(state)
+
+    def save(self, checkpoint_path: str):
+        """Storing current optimizer state."""
+        save_object = self.get_state()
+        with open(checkpoint_path, "wb") as f:
+            pickle.dump(save_object, f)
+
+    def restore(self, checkpoint_path: str):
+        """Restoring current optimizer state."""
+        with open(checkpoint_path, "rb") as f:
+            save_object = pickle.load(f)
+
+        if isinstance(save_object, dict):
+            self.set_state(save_object)
+        else:
+            # Backwards compatibility
+            (
+                self.optimizer,
+                self._buffered_trial_results,
+                self._total_random_search_trials,
+                self._config_counter,
+                self._points_to_evaluate,
+            ) = save_object
+
+    @staticmethod
+    def convert_search_space(spec: Dict, join: bool = False) -> Dict:
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a BayesOpt search space."
+            )
+
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        def resolve_value(domain: Domain) -> Tuple[float, float]:
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning(
+                    "BayesOpt search does not support quantization. "
+                    "Dropped quantization."
+                )
+                sampler = sampler.get_sampler()
+
+            if isinstance(domain, Float):
+                if domain.sampler is not None and not isinstance(
+                    domain.sampler, Uniform
+                ):
+                    logger.warning(
+                        "BayesOpt does not support specific sampling methods. "
+                        "The {} sampler will be dropped.".format(sampler)
+                    )
+                return (domain.lower, domain.upper)
+
+            raise ValueError(
+                "BayesOpt does not support parameters of type "
+                "`{}`".format(type(domain).__name__)
+            )
+
+        # Parameter name is e.g. "a/b/c" for nested dicts
+        bounds = {"/".join(path): resolve_value(domain) for path, domain in domain_vars}
+
+        if join:
+            spec.update(bounds)
+            bounds = spec
+
+        return bounds

python/ray/tune/search/bohb/__init__.py

@@ -0,0 +1,3 @@
+from ray.tune.search.bohb.bohb_search import BOHB, TuneBOHB
+
+__all__ = ["BOHB", "TuneBOHB"]

python/ray/tune/search/bohb/bohb_search.py

@@ -0,0 +1,379 @@
+"""BOHB (Bayesian Optimization with HyperBand)"""
+
+import copy
+import logging
+import math
+from typing import Dict, List, Optional, Union
+
+# use cloudpickle instead of pickle to make BOHB obj
+# pickleable
+from ray import cloudpickle
+from ray.tune.result import DEFAULT_METRIC
+from ray.tune.search import (
+    UNDEFINED_METRIC_MODE,
+    UNDEFINED_SEARCH_SPACE,
+    UNRESOLVED_SEARCH_SPACE,
+    Searcher,
+)
+from ray.tune.search.sample import (
+    Categorical,
+    Domain,
+    Float,
+    Integer,
+    LogUniform,
+    Normal,
+    Quantized,
+    Uniform,
+)
+from ray.tune.search.variant_generator import parse_spec_vars
+from ray.tune.utils.util import flatten_dict, unflatten_list_dict
+
+try:
+    import ConfigSpace
+    from hpbandster.optimizers.config_generators.bohb import BOHB
+except ImportError:
+    BOHB = ConfigSpace = None
+
+logger = logging.getLogger(__name__)
+
+
+class _BOHBJobWrapper:
+    """Mock object for HpBandSter to process."""
+
+    def __init__(self, loss: float, budget: float, config: Dict):
+        self.result = {"loss": loss}
+        self.kwargs = {"budget": budget, "config": config.copy()}
+        self.exception = None
+
+
+class TuneBOHB(Searcher):
+    """BOHB suggestion component.
+
+
+    Requires HpBandSter and ConfigSpace to be installed. You can install
+    HpBandSter and ConfigSpace with: ``pip install hpbandster ConfigSpace``.
+
+    This should be used in conjunction with HyperBandForBOHB.
+
+    Args:
+        space: Continuous ConfigSpace search space.
+            Parameters will be sampled from this space which will be used
+            to run trials.
+        bohb_config: configuration for HpBandSter BOHB algorithm
+        metric: The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
+        mode: One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute.
+        points_to_evaluate: Initial parameter suggestions to be run
+            first. This is for when you already have some good parameters
+            you want to run first to help the algorithm make better suggestions
+            for future parameters. Needs to be a list of dicts containing the
+            configurations.
+        seed: Optional random seed to initialize the random number
+            generator. Setting this should lead to identical initial
+            configurations at each run.
+        max_concurrent: Number of maximum concurrent trials.
+            If this Searcher is used in a ``ConcurrencyLimiter``, the
+            ``max_concurrent`` value passed to it will override the
+            value passed here. Set to <= 0 for no limit on concurrency.
+
+    Tune automatically converts search spaces to TuneBOHB's format:
+
+    .. code-block:: python
+
+        config = {
+            "width": tune.uniform(0, 20),
+            "height": tune.uniform(-100, 100),
+            "activation": tune.choice(["relu", "tanh"])
+        }
+
+        algo = TuneBOHB(metric="mean_loss", mode="min")
+        bohb = HyperBandForBOHB(
+            time_attr="training_iteration",
+            metric="mean_loss",
+            mode="min",
+            max_t=100)
+        run(my_trainable, config=config, scheduler=bohb, search_alg=algo)
+
+    If you would like to pass the search space manually, the code would
+    look like this:
+
+    .. code-block:: python
+
+        import ConfigSpace as CS
+
+        config_space = CS.ConfigurationSpace()
+        config_space.add_hyperparameter(
+            CS.UniformFloatHyperparameter("width", lower=0, upper=20))
+        config_space.add_hyperparameter(
+            CS.UniformFloatHyperparameter("height", lower=-100, upper=100))
+        config_space.add_hyperparameter(
+            CS.CategoricalHyperparameter(
+                name="activation", choices=["relu", "tanh"]))
+
+        algo = TuneBOHB(
+            config_space, metric="mean_loss", mode="min")
+        bohb = HyperBandForBOHB(
+            time_attr="training_iteration",
+            metric="mean_loss",
+            mode="min",
+            max_t=100)
+        run(my_trainable, scheduler=bohb, search_alg=algo)
+
+    """
+
+    def __init__(
+        self,
+        space: Optional[Union[Dict, "ConfigSpace.ConfigurationSpace"]] = None,
+        bohb_config: Optional[Dict] = None,
+        metric: Optional[str] = None,
+        mode: Optional[str] = None,
+        points_to_evaluate: Optional[List[Dict]] = None,
+        seed: Optional[int] = None,
+        max_concurrent: int = 0,
+    ):
+        assert (
+            BOHB is not None
+        ), """HpBandSter must be installed!
+            You can install HpBandSter with the command:
+            `pip install hpbandster ConfigSpace`."""
+        if mode:
+            assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
+        self.trial_to_params = {}
+        self._metric = metric
+
+        self._bohb_config = bohb_config
+
+        if isinstance(space, dict) and space:
+            resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
+            if domain_vars or grid_vars:
+                logger.warning(
+                    UNRESOLVED_SEARCH_SPACE.format(par="space", cls=type(self))
+                )
+                space = self.convert_search_space(space)
+
+        self._space = space
+        self._seed = seed
+
+        self.running = set()
+        self.paused = set()
+
+        self._max_concurrent = max_concurrent
+        self._points_to_evaluate = points_to_evaluate
+
+        super(TuneBOHB, self).__init__(
+            metric=self._metric,
+            mode=mode,
+        )
+
+        if self._space:
+            self._setup_bohb()
+
+    def set_max_concurrency(self, max_concurrent: int) -> bool:
+        self._max_concurrent = max_concurrent
+        return True
+
+    def _setup_bohb(self):
+        from hpbandster.optimizers.config_generators.bohb import BOHB
+
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
+        if self._mode == "max":
+            self._metric_op = -1.0
+        elif self._mode == "min":
+            self._metric_op = 1.0
+
+        if self._seed is not None:
+            self._space.seed(self._seed)
+
+        self.running = set()
+        self.paused = set()
+
+        bohb_config = self._bohb_config or {}
+        self.bohber = BOHB(self._space, **bohb_config)
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        if self._space:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        self._setup_bohb()
+        return True
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        if not self._space:
+            raise RuntimeError(
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"
+                )
+            )
+
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__, metric=self._metric, mode=self._mode
+                )
+            )
+
+        max_concurrent = (
+            self._max_concurrent if self._max_concurrent > 0 else float("inf")
+        )
+        if len(self.running) >= max_concurrent:
+            return None
+
+        if self._points_to_evaluate:
+            config = self._points_to_evaluate.pop(0)
+        else:
+            # This parameter is not used in hpbandster implementation.
+            config, _ = self.bohber.get_config(None)
+        self.trial_to_params[trial_id] = copy.deepcopy(config)
+        self.running.add(trial_id)
+        return unflatten_list_dict(config)
+
+    def on_trial_result(self, trial_id: str, result: Dict):
+        if trial_id not in self.paused:
+            self.running.add(trial_id)
+        if "hyperband_info" not in result:
+            logger.warning(
+                "BOHB Info not detected in result. Are you using "
+                "HyperBandForBOHB as a scheduler?"
+            )
+        elif "budget" in result.get("hyperband_info", {}):
+            hbs_wrapper = self.to_wrapper(trial_id, result)
+            self.bohber.new_result(hbs_wrapper)
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ):
+        del self.trial_to_params[trial_id]
+        self.paused.discard(trial_id)
+        self.running.discard(trial_id)
+
+    def to_wrapper(self, trial_id: str, result: Dict) -> _BOHBJobWrapper:
+        return _BOHBJobWrapper(
+            self._metric_op * result[self.metric],
+            result["hyperband_info"]["budget"],
+            self.trial_to_params[trial_id],
+        )
+
+    # BOHB Specific.
+    # TODO(team-ml): Refactor alongside HyperBandForBOHB
+    def on_pause(self, trial_id: str):
+        self.paused.add(trial_id)
+        self.running.discard(trial_id)
+
+    def on_unpause(self, trial_id: str):
+        self.paused.discard(trial_id)
+        self.running.add(trial_id)
+
+    @staticmethod
+    def convert_search_space(spec: Dict) -> "ConfigSpace.ConfigurationSpace":
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a TuneBOHB search space."
+            )
+
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        def resolve_value(
+            par: str, domain: Domain
+        ) -> ConfigSpace.hyperparameters.Hyperparameter:
+
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning(
+                    "TuneBOHB does not support quantization. "
+                    "Dropped quantization for parameter '%s'.",
+                    par,
+                )
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    return ConfigSpace.UniformFloatHyperparameter(
+                        par, lower=domain.lower, upper=domain.upper, log=True
+                    )
+                elif isinstance(sampler, Uniform):
+                    return ConfigSpace.UniformFloatHyperparameter(
+                        par, lower=domain.lower, upper=domain.upper, log=False
+                    )
+                elif isinstance(sampler, Normal):
+                    if (
+                        domain.lower is None
+                        or domain.upper is None
+                        or not math.isfinite(domain.lower)
+                        or not math.isfinite(domain.upper)
+                    ):
+                        raise ValueError(
+                            f"TuneBOHB does not support unbounded normal "
+                            f"distributions. Please specify bounds for "
+                            f"parameter '{par}' using tune.randn(...).clip(lower, upper) "
+                            f"or Float(lower, upper).normal(mean, sd)."
+                        )
+                    return ConfigSpace.hyperparameters.NormalFloatHyperparameter(
+                        par,
+                        mu=sampler.mean,
+                        sigma=sampler.sd,
+                        lower=domain.lower,
+                        upper=domain.upper,
+                        log=False,
+                    )
+
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, LogUniform):
+                    # Tune search space integers are exclusive on upper bound
+                    return ConfigSpace.UniformIntegerHyperparameter(
+                        par, lower=domain.lower, upper=domain.upper - 1, log=True
+                    )
+                elif isinstance(sampler, Uniform):
+                    # Tune search space integers are exclusive on upper bound
+                    return ConfigSpace.UniformIntegerHyperparameter(
+                        par, lower=domain.lower, upper=domain.upper - 1, log=False
+                    )
+
+            elif isinstance(domain, Categorical):
+                if isinstance(sampler, Uniform):
+                    return ConfigSpace.CategoricalHyperparameter(
+                        par, choices=domain.categories
+                    )
+
+            raise ValueError(
+                "TuneBOHB does not support parameters of type "
+                "`{}` with samplers of type `{}`".format(
+                    type(domain).__name__, type(domain.sampler).__name__
+                )
+            )
+
+        cs = ConfigSpace.ConfigurationSpace()
+        for path, domain in domain_vars:
+            par = "/".join(str(p) for p in path)
+            value = resolve_value(par, domain)
+            cs.add_hyperparameter(value)
+
+        return cs
+
+    def save(self, checkpoint_path: str):
+        save_object = self.__dict__
+        with open(checkpoint_path, "wb") as outputFile:
+            cloudpickle.dump(save_object, outputFile)
+
+    def restore(self, checkpoint_path: str):
+        with open(checkpoint_path, "rb") as inputFile:
+            save_object = cloudpickle.load(inputFile)
+        self.__dict__.update(save_object)

python/ray/tune/search/concurrency_limiter.py

@@ -0,0 +1,176 @@
+import copy
+import logging
+from typing import Dict, List, Optional
+
+from ray.tune.search.searcher import Searcher
+from ray.tune.search.util import _set_search_properties_backwards_compatible
+from ray.util.annotations import PublicAPI
+
+logger = logging.getLogger(__name__)
+
+
+@PublicAPI
+class ConcurrencyLimiter(Searcher):
+    """A wrapper algorithm for limiting the number of concurrent trials.
+
+    Certain Searchers have their own internal logic for limiting
+    the number of concurrent trials. If such a Searcher is passed to a
+    ``ConcurrencyLimiter``, the ``max_concurrent`` of the
+    ``ConcurrencyLimiter`` will override the ``max_concurrent`` value
+    of the Searcher. The ``ConcurrencyLimiter`` will then let the
+    Searcher's internal logic take over.
+
+    Args:
+        searcher: Searcher object that the
+            ConcurrencyLimiter will manage.
+        max_concurrent: Maximum concurrent samples from the underlying
+            searcher.
+        batch: Whether to wait for all concurrent samples
+            to finish before updating the underlying searcher.
+
+    Example:
+
+    .. code-block:: python
+
+        from ray.tune.search import ConcurrencyLimiter
+        search_alg = HyperOptSearch(metric="accuracy")
+        search_alg = ConcurrencyLimiter(search_alg, max_concurrent=2)
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=search_alg
+            ),
+        )
+        tuner.fit()
+    """
+
+    def __init__(self, searcher: Searcher, max_concurrent: int, batch: bool = False):
+        assert isinstance(max_concurrent, int) and max_concurrent > 0
+        self.searcher = searcher
+        self.max_concurrent = max_concurrent
+        self.batch = batch
+        self.live_trials = set()
+        self.num_unfinished_live_trials = 0
+        self.cached_results = {}
+        self._limit_concurrency = True
+
+        if not isinstance(searcher, Searcher):
+            raise RuntimeError(
+                f"The `ConcurrencyLimiter` only works with `Searcher` "
+                f"objects (got {type(searcher)}). Please try to pass "
+                f"`max_concurrent` to the search generator directly."
+            )
+
+        self._set_searcher_max_concurrency()
+
+        super(ConcurrencyLimiter, self).__init__(
+            metric=self.searcher.metric, mode=self.searcher.mode
+        )
+
+    def _set_searcher_max_concurrency(self):
+        # If the searcher has special logic for handling max concurrency,
+        # we do not do anything inside the ConcurrencyLimiter
+        self._limit_concurrency = not self.searcher.set_max_concurrency(
+            self.max_concurrent
+        )
+
+    def set_max_concurrency(self, max_concurrent: int) -> bool:
+        # Determine if this behavior is acceptable, or if it should
+        # raise an exception.
+        self.max_concurrent = max_concurrent
+        return True
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        self._set_searcher_max_concurrency()
+        return _set_search_properties_backwards_compatible(
+            self.searcher.set_search_properties, metric, mode, config, **spec
+        )
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        if not self._limit_concurrency:
+            return self.searcher.suggest(trial_id)
+
+        assert (
+            trial_id not in self.live_trials
+        ), f"Trial ID {trial_id} must be unique: already found in set."
+        if len(self.live_trials) >= self.max_concurrent:
+            logger.debug(
+                f"Not providing a suggestion for {trial_id} due to "
+                "concurrency limit: %s/%s.",
+                len(self.live_trials),
+                self.max_concurrent,
+            )
+            return
+
+        suggestion = self.searcher.suggest(trial_id)
+        if suggestion not in (None, Searcher.FINISHED):
+            self.live_trials.add(trial_id)
+            self.num_unfinished_live_trials += 1
+        return suggestion
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ):
+        if not self._limit_concurrency:
+            return self.searcher.on_trial_complete(trial_id, result=result, error=error)
+
+        if trial_id not in self.live_trials:
+            return
+        elif self.batch:
+            self.cached_results[trial_id] = (result, error)
+            self.num_unfinished_live_trials -= 1
+            if self.num_unfinished_live_trials <= 0:
+                # Update the underlying searcher once the
+                # full batch is completed.
+                for trial_id, (result, error) in self.cached_results.items():
+                    self.searcher.on_trial_complete(
+                        trial_id, result=result, error=error
+                    )
+                    self.live_trials.remove(trial_id)
+                self.cached_results = {}
+                self.num_unfinished_live_trials = 0
+            else:
+                return
+        else:
+            self.searcher.on_trial_complete(trial_id, result=result, error=error)
+            self.live_trials.remove(trial_id)
+            self.num_unfinished_live_trials -= 1
+
+    def on_trial_result(self, trial_id: str, result: Dict) -> None:
+        self.searcher.on_trial_result(trial_id, result)
+
+    def add_evaluated_point(
+        self,
+        parameters: Dict,
+        value: float,
+        error: bool = False,
+        pruned: bool = False,
+        intermediate_values: Optional[List[float]] = None,
+    ):
+        return self.searcher.add_evaluated_point(
+            parameters, value, error, pruned, intermediate_values
+        )
+
+    def get_state(self) -> Dict:
+        state = self.__dict__.copy()
+        del state["searcher"]
+        return copy.deepcopy(state)
+
+    def set_state(self, state: Dict):
+        self.__dict__.update(state)
+
+    def save(self, checkpoint_path: str):
+        self.searcher.save(checkpoint_path)
+
+    def restore(self, checkpoint_path: str):
+        self.searcher.restore(checkpoint_path)
+
+    # BOHB Specific.
+    # TODO(team-ml): Refactor alongside HyperBandForBOHB
+    def on_pause(self, trial_id: str):
+        self.searcher.on_pause(trial_id)
+
+    def on_unpause(self, trial_id: str):
+        self.searcher.on_unpause(trial_id)

python/ray/tune/search/hebo/__init__.py

@@ -0,0 +1,3 @@
+from ray.tune.search.hebo.hebo_search import HEBOSearch
+
+__all__ = ["HEBOSearch"]

python/ray/tune/search/hebo/hebo_search.py

@@ -0,0 +1,466 @@
+import logging
+import pickle
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from ray.tune.result import DEFAULT_METRIC
+from ray.tune.search import (
+    UNDEFINED_METRIC_MODE,
+    UNDEFINED_SEARCH_SPACE,
+    UNRESOLVED_SEARCH_SPACE,
+    Searcher,
+)
+from ray.tune.search.sample import (
+    Categorical,
+    Domain,
+    Float,
+    Integer,
+    LogUniform,
+    Quantized,
+    Uniform,
+)
+from ray.tune.search.variant_generator import parse_spec_vars
+from ray.tune.utils.util import is_nan_or_inf, unflatten_dict, validate_warmstart
+
+try:  # Python 3 only -- needed for lint test.
+    import hebo
+    import torch  # hebo has torch as a dependency
+except ImportError:
+    hebo = None
+
+logger = logging.getLogger(__name__)
+
+SPACE_ERROR_MESSAGE = (
+    "Space must be either a HEBO DesignSpace object"
+    "or a dictionary with ONLY tune search spaces."
+)
+
+
+class HEBOSearch(Searcher):
+    """Uses HEBO (Heteroscedastic Evolutionary Bayesian Optimization)
+    to optimize hyperparameters.
+
+    HEBO is a cutting edge black-box optimization framework created
+    by Huawei's Noah Ark. More info can be found here:
+    https://github.com/huawei-noah/HEBO/tree/master/HEBO.
+
+    `space` can either be a HEBO's `DesignSpace` object or a dict of Tune
+    search spaces.
+
+    Please note that the first few trials will be random and used
+    to kickstart the search process. In order to achieve good results,
+    we recommend setting the number of trials to at least 16.
+
+    Maximum number of concurrent trials is determined by ``max_concurrent``
+    argument. Trials will be done in batches of ``max_concurrent`` trials.
+    If this Searcher is used in a ``ConcurrencyLimiter``, the
+    ``max_concurrent`` value passed to it will override the value passed
+    here.
+
+    Args:
+        space: A dict mapping parameter names to Tune search spaces or a
+            HEBO DesignSpace object.
+        metric: The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
+        mode: One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute.
+        points_to_evaluate: Initial parameter suggestions to be run
+            first. This is for when you already have some good parameters
+            you want to run first to help the algorithm make better suggestions
+            for future parameters. Needs to be a list of dicts containing the
+            configurations.
+        evaluated_rewards: If you have previously evaluated the
+            parameters passed in as points_to_evaluate you can avoid
+            re-running those trials by passing in the reward attributes
+            as a list so the optimiser can be told the results without
+            needing to re-compute the trial. Must be the same length as
+            points_to_evaluate.
+        random_state_seed: Seed for reproducible
+            results. Defaults to None. Please note that setting this to a value
+            will change global random states for `numpy` and `torch`
+            on initalization and loading from checkpoint.
+        max_concurrent: Number of maximum concurrent trials.
+            If this Searcher is used in a ``ConcurrencyLimiter``, the
+            ``max_concurrent`` value passed to it will override the
+            value passed here.
+        **kwargs: The keyword arguments will be passed to `HEBO()``.
+
+    Tune automatically converts search spaces to HEBO's format:
+
+    .. code-block:: python
+
+        from ray import tune
+        from ray.tune.search.hebo import HEBOSearch
+
+        config = {
+            "width": tune.uniform(0, 20),
+            "height": tune.uniform(-100, 100)
+        }
+
+        hebo = HEBOSearch(metric="mean_loss", mode="min")
+        tuner = tune.Tuner(
+            trainable_function,
+            tune_config=tune.TuneConfig(
+                search_alg=hebo
+            ),
+            param_space=config
+        )
+        tuner.fit()
+
+    Alternatively, you can pass a HEBO `DesignSpace` object manually to the
+    Searcher:
+
+    .. code-block:: python
+
+        from ray import tune
+        from ray.tune.search.hebo import HEBOSearch
+        from hebo.design_space.design_space import DesignSpace
+
+        space_config = [
+            {'name' : 'width', 'type' : 'num', 'lb' : 0, 'ub' : 20},
+            {'name' : 'height', 'type' : 'num', 'lb' : -100, 'ub' : 100},
+        ]
+        space = DesignSpace().parse(space_config)
+
+        hebo = HEBOSearch(space, metric="mean_loss", mode="min")
+        tuner = tune.Tuner(
+            trainable_function,
+            tune_config=tune.TuneConfig(
+                search_alg=hebo
+            )
+        )
+        tuner.fit()
+
+    """
+
+    def __init__(
+        self,
+        space: Optional[
+            Union[Dict, "hebo.design_space.design_space.DesignSpace"]
+        ] = None,
+        metric: Optional[str] = None,
+        mode: Optional[str] = None,
+        points_to_evaluate: Optional[List[Dict]] = None,
+        evaluated_rewards: Optional[List] = None,
+        random_state_seed: Optional[int] = None,
+        max_concurrent: int = 8,
+        **kwargs,
+    ):
+        assert hebo is not None, (
+            "HEBO must be installed! You can install HEBO with"
+            " the command: `pip install 'HEBO>=0.2.0'`."
+            "This error may also be caused if HEBO"
+            " dependencies have bad versions. Try updating HEBO"
+            " first."
+        )
+        if mode:
+            assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
+        assert (
+            isinstance(max_concurrent, int) and max_concurrent >= 1
+        ), "`max_concurrent` must be an integer and at least 1."
+        if random_state_seed is not None:
+            assert isinstance(
+                random_state_seed, int
+            ), "random_state_seed must be None or int, got '{}'.".format(
+                type(random_state_seed)
+            )
+        super(HEBOSearch, self).__init__(metric=metric, mode=mode)
+
+        if isinstance(space, dict) and space:
+            resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
+            if resolved_vars:
+                raise TypeError(SPACE_ERROR_MESSAGE)
+            if domain_vars or grid_vars:
+                logger.warning(
+                    UNRESOLVED_SEARCH_SPACE.format(par="space", cls=type(self))
+                )
+                space = self.convert_search_space(space)
+        elif space is not None and not isinstance(
+            space, hebo.design_space.design_space.DesignSpace
+        ):
+            raise TypeError(SPACE_ERROR_MESSAGE + " Got {}.".format(type(space)))
+
+        self._hebo_config = kwargs
+        self._random_state_seed = random_state_seed
+        self._space = space
+        self._points_to_evaluate = points_to_evaluate
+        self._evaluated_rewards = evaluated_rewards
+        self._initial_points = []
+        self._live_trial_mapping = {}
+
+        self._max_concurrent = max_concurrent
+        self._suggestions_cache = []
+        self._batch_filled = False
+
+        self._opt = None
+        if space:
+            self._setup_optimizer()
+
+    def set_max_concurrency(self, max_concurrent: int) -> bool:
+        self._max_concurrent = max_concurrent
+        return True
+
+    def _setup_optimizer(self):
+        # HEBO internally minimizes, so "max" => -1
+        if self._mode == "max":
+            self._metric_op = -1.0
+        elif self._mode == "min":
+            self._metric_op = 1.0
+
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
+        if not isinstance(self._space, hebo.design_space.design_space.DesignSpace):
+            raise ValueError(
+                f"Invalid search space: {type(self._space)}. Either pass a "
+                f"valid search space to the `HEBOSearch` class or pass "
+                f"a `param_space` parameter to `tune.Tuner()`"
+            )
+
+        if self._space.num_paras <= 0:
+            raise ValueError(
+                "Got empty search space. Please make sure to pass "
+                "a valid search space with at least one parameter to "
+                "`HEBOSearch`"
+            )
+
+        if self._random_state_seed is not None:
+            np.random.seed(self._random_state_seed)
+            torch.random.manual_seed(self._random_state_seed)
+
+        self._opt = hebo.optimizers.hebo.HEBO(space=self._space, **self._hebo_config)
+
+        if self._points_to_evaluate:
+            validate_warmstart(
+                self._space.para_names,
+                self._points_to_evaluate,
+                self._evaluated_rewards,
+            )
+            if self._evaluated_rewards:
+                self._opt.observe(
+                    pd.DataFrame(self._points_to_evaluate),
+                    np.array(self._evaluated_rewards) * self._metric_op,
+                )
+            else:
+                self._initial_points = self._points_to_evaluate
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        if self._opt:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        self._setup_optimizer()
+        return True
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        if not self._opt:
+            raise RuntimeError(
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"
+                )
+            )
+
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__, metric=self._metric, mode=self._mode
+                )
+            )
+
+        if not self._live_trial_mapping:
+            self._batch_filled = False
+
+        if self._initial_points:
+            params = self._initial_points.pop(0)
+            suggestion = pd.DataFrame([params], index=[0])
+        else:
+            if (
+                self._batch_filled
+                or len(self._live_trial_mapping) >= self._max_concurrent
+            ):
+                return None
+            if not self._suggestions_cache:
+                suggestion = self._opt.suggest(n_suggestions=self._max_concurrent)
+                self._suggestions_cache = suggestion.to_dict("records")
+            params = self._suggestions_cache.pop(0)
+            suggestion = pd.DataFrame([params], index=[0])
+        self._live_trial_mapping[trial_id] = suggestion
+        if len(self._live_trial_mapping) >= self._max_concurrent:
+            self._batch_filled = True
+        return unflatten_dict(params)
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ):
+        """Notification for the completion of trial.
+
+        HEBO always minimizes."""
+
+        if result:
+            self._process_result(trial_id, result)
+        self._live_trial_mapping.pop(trial_id)
+
+    def _process_result(self, trial_id: str, result: Dict):
+        trial_info = self._live_trial_mapping[trial_id]
+        if result and not is_nan_or_inf(result[self._metric]):
+            self._opt.observe(
+                trial_info, np.array([self._metric_op * result[self._metric]])
+            )
+
+    def add_evaluated_point(
+        self,
+        parameters: Dict,
+        value: float,
+        error: bool = False,
+        pruned: bool = False,
+        intermediate_values: Optional[List[float]] = None,
+    ):
+        if intermediate_values:
+            logger.warning("HEBO doesn't use intermediate_values. Ignoring.")
+        if not error and not pruned:
+            self._opt.observe(
+                pd.DataFrame(
+                    [
+                        {
+                            k: v
+                            for k, v in parameters.items()
+                            if k in self._opt.space.para_names
+                        }
+                    ]
+                ),
+                np.array([value]) * self._metric_op,
+            )
+        else:
+            logger.warning(
+                "Only non errored and non pruned points can be added to HEBO."
+            )
+
+    def save(self, checkpoint_path: str):
+        """Storing current optimizer state."""
+        if self._random_state_seed is not None:
+            numpy_random_state = np.random.get_state()
+            torch_random_state = torch.get_rng_state()
+        else:
+            numpy_random_state = None
+            torch_random_state = None
+        save_object = self.__dict__.copy()
+        save_object["__numpy_random_state"] = numpy_random_state
+        save_object["__torch_random_state"] = torch_random_state
+        with open(checkpoint_path, "wb") as f:
+            pickle.dump(save_object, f)
+
+    def restore(self, checkpoint_path: str):
+        """Restoring current optimizer state."""
+        with open(checkpoint_path, "rb") as f:
+            save_object = pickle.load(f)
+
+        if isinstance(save_object, dict):
+            numpy_random_state = save_object.pop("__numpy_random_state", None)
+            torch_random_state = save_object.pop("__torch_random_state", None)
+            self.__dict__.update(save_object)
+        else:
+            # Backwards compatibility
+            (
+                self._opt,
+                self._initial_points,
+                numpy_random_state,
+                torch_random_state,
+                self._live_trial_mapping,
+                self._max_concurrent,
+                self._suggestions_cache,
+                self._space,
+                self._hebo_config,
+                self._batch_filled,
+            ) = save_object
+        if numpy_random_state is not None:
+            np.random.set_state(numpy_random_state)
+        if torch_random_state is not None:
+            torch.random.set_rng_state(torch_random_state)
+
+    @staticmethod
+    def convert_search_space(spec: Dict, prefix: str = "") -> Dict:
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        params = []
+
+        if not domain_vars and not grid_vars:
+            return {}
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a HEBO search space."
+            )
+
+        def resolve_value(par: str, domain: Domain):
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning(
+                    "HEBO search does not support quantization. "
+                    "Dropped quantization."
+                )
+                sampler = sampler.get_sampler()
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    return {
+                        "name": par,
+                        "type": "pow",
+                        "lb": domain.lower,
+                        "ub": domain.upper,
+                    }
+                elif isinstance(sampler, Uniform):
+                    return {
+                        "name": par,
+                        "type": "num",
+                        "lb": domain.lower,
+                        "ub": domain.upper,
+                    }
+
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, LogUniform):
+                    return {
+                        "name": par,
+                        "type": "pow_int",
+                        "lb": domain.lower,
+                        "ub": domain.upper - 1,  # Upper bound exclusive
+                    }
+                elif isinstance(sampler, Uniform):
+                    return {
+                        "name": par,
+                        "type": "int",
+                        "lb": domain.lower,
+                        "ub": domain.upper - 1,  # Upper bound exclusive
+                    }
+            elif isinstance(domain, Categorical):
+                return {
+                    "name": par,
+                    "type": "cat",
+                    "categories": list(domain.categories),
+                }
+
+            raise ValueError(
+                "HEBO does not support parameters of type "
+                "`{}` with samplers of type `{}`".format(
+                    type(domain).__name__, type(domain.sampler).__name__
+                )
+            )
+
+        for path, domain in domain_vars:
+            par = "/".join([str(p) for p in ((prefix,) + path if prefix else path)])
+            value = resolve_value(par, domain)
+            params.append(value)
+
+        return hebo.design_space.design_space.DesignSpace().parse(params)

python/ray/tune/search/hyperopt/__init__.py

@@ -0,0 +1,3 @@
+from ray.tune.search.hyperopt.hyperopt_search import HyperOptSearch
+
+__all__ = ["HyperOptSearch"]

python/ray/tune/search/hyperopt/hyperopt_search.py

@@ -0,0 +1,559 @@
+import copy
+import logging
+from functools import partial
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+
+# Use cloudpickle instead of pickle to make lambda funcs in HyperOpt pickleable
+from ray import cloudpickle
+from ray.tune.error import TuneError
+from ray.tune.result import DEFAULT_METRIC
+from ray.tune.search import (
+    UNDEFINED_METRIC_MODE,
+    UNDEFINED_SEARCH_SPACE,
+    UNRESOLVED_SEARCH_SPACE,
+    Searcher,
+)
+from ray.tune.search.sample import (
+    Categorical,
+    Domain,
+    Float,
+    Integer,
+    LogUniform,
+    Normal,
+    Quantized,
+    Uniform,
+)
+from ray.tune.search.variant_generator import assign_value, parse_spec_vars
+from ray.tune.utils import flatten_dict
+
+try:
+    hyperopt_logger = logging.getLogger("hyperopt")
+    hyperopt_logger.setLevel(logging.WARNING)
+    import hyperopt as hpo
+    from hyperopt.pyll import Apply
+except ImportError:
+    hpo = None
+    Apply = None
+
+
+logger = logging.getLogger(__name__)
+
+
+HYPEROPT_UNDEFINED_DETAILS = (
+    " This issue can also come up with HyperOpt if your search space only "
+    "contains constant variables, which is not supported by HyperOpt. In that case, "
+    "don't pass any searcher or add sample variables to the search space."
+)
+
+
+class HyperOptSearch(Searcher):
+    """A wrapper around HyperOpt to provide trial suggestions.
+
+    HyperOpt a Python library for serial and parallel optimization
+    over awkward search spaces, which may include real-valued, discrete,
+    and conditional dimensions. More info can be found at
+    http://hyperopt.github.io/hyperopt.
+
+    HyperOptSearch uses the Tree-structured Parzen Estimators algorithm,
+    though it can be trivially extended to support any algorithm HyperOpt
+    supports.
+
+    To use this search algorithm, you will need to install HyperOpt:
+
+    .. code-block:: bash
+
+        pip install -U hyperopt
+
+
+    Parameters:
+        space: HyperOpt configuration. Parameters will be sampled
+            from this configuration and will be used to override
+            parameters generated in the variant generation process.
+        metric: The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
+        mode: One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute.
+        points_to_evaluate: Initial parameter suggestions to be run
+            first. This is for when you already have some good parameters
+            you want to run first to help the algorithm make better suggestions
+            for future parameters. Needs to be a list of dicts containing the
+            configurations.
+        n_initial_points: number of random evaluations of the
+            objective function before starting to approximate it with
+            tree parzen estimators. Defaults to 20.
+        random_state_seed: seed for reproducible
+            results. Defaults to None.
+        gamma: parameter governing the tree parzen
+            estimators suggestion algorithm. Defaults to 0.25.
+
+    Tune automatically converts search spaces to HyperOpt's format:
+
+    .. code-block:: python
+
+        config = {
+            'width': tune.uniform(0, 20),
+            'height': tune.uniform(-100, 100),
+            'activation': tune.choice(["relu", "tanh"])
+        }
+
+        current_best_params = [{
+            'width': 10,
+            'height': 0,
+            'activation': "relu",
+        }]
+
+        hyperopt_search = HyperOptSearch(
+            metric="mean_loss", mode="min",
+            points_to_evaluate=current_best_params)
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=hyperopt_search
+            ),
+            param_space=config
+        )
+        tuner.fit()
+
+    If you would like to pass the search space manually, the code would
+    look like this:
+
+    .. code-block:: python
+
+        space = {
+            'width': hp.uniform('width', 0, 20),
+            'height': hp.uniform('height', -100, 100),
+            'activation': hp.choice("activation", ["relu", "tanh"])
+        }
+
+        current_best_params = [{
+            'width': 10,
+            'height': 0,
+            'activation': "relu",
+        }]
+
+        hyperopt_search = HyperOptSearch(
+            space, metric="mean_loss", mode="min",
+            points_to_evaluate=current_best_params)
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=hyperopt_search
+            ),
+        )
+        tuner.fit()
+
+    """
+
+    def __init__(
+        self,
+        space: Optional[Dict] = None,
+        metric: Optional[str] = None,
+        mode: Optional[str] = None,
+        points_to_evaluate: Optional[List[Dict]] = None,
+        n_initial_points: int = 20,
+        random_state_seed: Optional[int] = None,
+        gamma: float = 0.25,
+    ):
+        assert (
+            hpo is not None
+        ), "HyperOpt must be installed! Run `pip install hyperopt`."
+        if mode:
+            assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
+        super(HyperOptSearch, self).__init__(
+            metric=metric,
+            mode=mode,
+        )
+        # hyperopt internally minimizes, so "max" => -1
+        if mode == "max":
+            self.metric_op = -1.0
+        elif mode == "min":
+            self.metric_op = 1.0
+
+        if n_initial_points is None:
+            self.algo = hpo.tpe.suggest
+        else:
+            self.algo = partial(hpo.tpe.suggest, n_startup_jobs=n_initial_points)
+        if gamma is not None:
+            self.algo = partial(self.algo, gamma=gamma)
+
+        self._points_to_evaluate = copy.deepcopy(points_to_evaluate)
+
+        self._live_trial_mapping = {}
+        self.rstate = np.random.RandomState(random_state_seed)
+
+        self.domain = None
+        if isinstance(space, dict) and space:
+            resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
+            if domain_vars or grid_vars:
+                logger.warning(
+                    UNRESOLVED_SEARCH_SPACE.format(par="space", cls=type(self))
+                )
+                space = self.convert_search_space(space)
+            self._space = space
+            self._setup_hyperopt()
+
+    def _setup_hyperopt(self) -> None:
+        from hyperopt.fmin import generate_trials_to_calculate
+
+        if not self._space:
+            raise RuntimeError(
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"
+                )
+                + HYPEROPT_UNDEFINED_DETAILS
+            )
+
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
+        if self._points_to_evaluate is None:
+            self._hpopt_trials = hpo.Trials()
+            self._points_to_evaluate = 0
+        else:
+            assert isinstance(self._points_to_evaluate, (list, tuple))
+
+            for i in range(len(self._points_to_evaluate)):
+                config = self._points_to_evaluate[i]
+                self._convert_categories_to_indices(config)
+            # HyperOpt treats initial points as LIFO, reverse to get FIFO
+            self._points_to_evaluate = list(reversed(self._points_to_evaluate))
+            self._hpopt_trials = generate_trials_to_calculate(self._points_to_evaluate)
+            self._hpopt_trials.refresh()
+            self._points_to_evaluate = len(self._points_to_evaluate)
+
+        self.domain = hpo.Domain(lambda spc: spc, self._space)
+
+    def _convert_categories_to_indices(self, config) -> None:
+        """Convert config parameters for categories into hyperopt-compatible
+        representations where instead the index of the category is expected."""
+
+        def _lookup(config_dict, space_dict, key):
+            if isinstance(config_dict[key], dict):
+                for k in config_dict[key]:
+                    _lookup(config_dict[key], space_dict[key], k)
+            else:
+                if (
+                    key in space_dict
+                    and isinstance(space_dict[key], hpo.base.pyll.Apply)
+                    and space_dict[key].name == "switch"
+                ):
+                    if len(space_dict[key].pos_args) > 0:
+                        categories = [
+                            a.obj
+                            for a in space_dict[key].pos_args[1:]
+                            if a.name == "literal"
+                        ]
+                        try:
+                            idx = categories.index(config_dict[key])
+                        except ValueError as exc:
+                            msg = (
+                                f"Did not find category with value "
+                                f"`{config_dict[key]}` in "
+                                f"hyperopt parameter `{key}`. "
+                            )
+
+                            if isinstance(config_dict[key], int):
+                                msg += (
+                                    "In previous versions, a numerical "
+                                    "index was expected for categorical "
+                                    "values of `points_to_evaluate`, "
+                                    "but in ray>=1.2.0, the categorical "
+                                    "value is expected to be directly "
+                                    "provided. "
+                                )
+
+                            msg += "Please make sure the specified category is valid."
+                            raise ValueError(msg) from exc
+                        config_dict[key] = idx
+
+        for k in config:
+            _lookup(config, self._space, k)
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        if self.domain:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        self.metric_op = -1.0 if self._mode == "max" else 1.0
+
+        self._setup_hyperopt()
+        return True
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        if not self.domain:
+            raise RuntimeError(
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"
+                )
+                + HYPEROPT_UNDEFINED_DETAILS
+            )
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__, metric=self._metric, mode=self._mode
+                )
+            )
+
+        if self._points_to_evaluate > 0:
+            using_point_to_evaluate = True
+            new_trial = self._hpopt_trials.trials[self._points_to_evaluate - 1]
+            self._points_to_evaluate -= 1
+        else:
+            using_point_to_evaluate = False
+            new_ids = self._hpopt_trials.new_trial_ids(1)
+            self._hpopt_trials.refresh()
+
+            # Get new suggestion from Hyperopt
+            new_trials = self.algo(
+                new_ids,
+                self.domain,
+                self._hpopt_trials,
+                self.rstate.randint(2**31 - 1),
+            )
+            self._hpopt_trials.insert_trial_docs(new_trials)
+            self._hpopt_trials.refresh()
+            new_trial = new_trials[0]
+        self._live_trial_mapping[trial_id] = (new_trial["tid"], new_trial)
+
+        # Taken from HyperOpt.base.evaluate
+        config = hpo.base.spec_from_misc(new_trial["misc"])
+
+        # We have to flatten nested spaces here so parameter names match
+        config = flatten_dict(config, flatten_list=True)
+
+        ctrl = hpo.base.Ctrl(self._hpopt_trials, current_trial=new_trial)
+        memo = self.domain.memo_from_config(config)
+        hpo.utils.use_obj_for_literal_in_memo(
+            self.domain.expr, ctrl, hpo.base.Ctrl, memo
+        )
+
+        try:
+            suggested_config = hpo.pyll.rec_eval(
+                self.domain.expr,
+                memo=memo,
+                print_node_on_error=self.domain.rec_eval_print_node_on_error,
+            )
+        except (AssertionError, TypeError) as e:
+            if using_point_to_evaluate and (
+                isinstance(e, AssertionError) or "GarbageCollected" in str(e)
+            ):
+                raise ValueError(
+                    "HyperOpt encountered a GarbageCollected switch argument. "
+                    "Usually this is caused by a config in "
+                    "`points_to_evaluate` "
+                    "missing a key present in `space`. Ensure that "
+                    "`points_to_evaluate` contains "
+                    "all non-constant keys from `space`.\n"
+                    "Config from `points_to_evaluate`: "
+                    f"{config}\n"
+                    "HyperOpt search space: "
+                    f"{self._space}"
+                ) from e
+            raise e
+        return copy.deepcopy(suggested_config)
+
+    def on_trial_result(self, trial_id: str, result: Dict) -> None:
+        ho_trial = self._get_hyperopt_trial(trial_id)
+        if ho_trial is None:
+            return
+        now = hpo.utils.coarse_utcnow()
+        ho_trial["book_time"] = now
+        ho_trial["refresh_time"] = now
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ) -> None:
+        """Notification for the completion of trial.
+
+        The result is internally negated when interacting with HyperOpt
+        so that HyperOpt can "maximize" this value, as it minimizes on default.
+        """
+        ho_trial = self._get_hyperopt_trial(trial_id)
+        if ho_trial is None:
+            return
+        ho_trial["refresh_time"] = hpo.utils.coarse_utcnow()
+        if error:
+            ho_trial["state"] = hpo.base.JOB_STATE_ERROR
+            ho_trial["misc"]["error"] = (str(TuneError), "Tune Error")
+            self._hpopt_trials.refresh()
+        elif result:
+            self._process_result(trial_id, result)
+        del self._live_trial_mapping[trial_id]
+
+    def _process_result(self, trial_id: str, result: Dict) -> None:
+        ho_trial = self._get_hyperopt_trial(trial_id)
+        if not ho_trial:
+            return
+        ho_trial["refresh_time"] = hpo.utils.coarse_utcnow()
+
+        ho_trial["state"] = hpo.base.JOB_STATE_DONE
+        hp_result = self._to_hyperopt_result(result)
+        ho_trial["result"] = hp_result
+        self._hpopt_trials.refresh()
+
+    def _to_hyperopt_result(self, result: Dict) -> Dict:
+        try:
+            return {"loss": self.metric_op * result[self.metric], "status": "ok"}
+        except KeyError as e:
+            raise RuntimeError(
+                f"Hyperopt expected to see the metric `{self.metric}` in the "
+                f"last result, but it was not found. To fix this, make "
+                f"sure your call to `tune.report` or your return value of "
+                f"your trainable class `step()` contains the above metric "
+                f"as a key."
+            ) from e
+
+    def _get_hyperopt_trial(self, trial_id: str) -> Optional[Dict]:
+        if trial_id not in self._live_trial_mapping:
+            return
+        hyperopt_tid = self._live_trial_mapping[trial_id][0]
+        return [t for t in self._hpopt_trials.trials if t["tid"] == hyperopt_tid][0]
+
+    def get_state(self) -> Dict:
+        return {
+            "hyperopt_trials": self._hpopt_trials,
+            "rstate": self.rstate.get_state(),
+        }
+
+    def set_state(self, state: Dict) -> None:
+        self._hpopt_trials = state["hyperopt_trials"]
+        self.rstate.set_state(state["rstate"])
+
+    def save(self, checkpoint_path: str) -> None:
+        save_object = self.__dict__.copy()
+        save_object["__rstate"] = self.rstate.get_state()
+        with open(checkpoint_path, "wb") as f:
+            cloudpickle.dump(save_object, f)
+
+    def restore(self, checkpoint_path: str) -> None:
+        with open(checkpoint_path, "rb") as f:
+            save_object = cloudpickle.load(f)
+
+        if "__rstate" not in save_object:
+            # Backwards compatibility
+            self.set_state(save_object)
+        else:
+            self.rstate.set_state(save_object.pop("__rstate"))
+            self.__dict__.update(save_object)
+
+    @staticmethod
+    def convert_search_space(spec: Dict, prefix: str = "") -> Dict:
+        spec = copy.deepcopy(spec)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if not domain_vars and not grid_vars:
+            return {}
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a HyperOpt search space."
+            )
+
+        def resolve_value(par: str, domain: Domain) -> Any:
+            quantize = None
+
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                quantize = sampler.q
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    if quantize:
+                        return hpo.hp.qloguniform(
+                            par, np.log(domain.lower), np.log(domain.upper), quantize
+                        )
+                    return hpo.hp.loguniform(
+                        par, np.log(domain.lower), np.log(domain.upper)
+                    )
+                elif isinstance(sampler, Uniform):
+                    if quantize:
+                        return hpo.hp.quniform(
+                            par, domain.lower, domain.upper, quantize
+                        )
+                    return hpo.hp.uniform(par, domain.lower, domain.upper)
+                elif isinstance(sampler, Normal):
+                    if quantize:
+                        return hpo.hp.qnormal(par, sampler.mean, sampler.sd, quantize)
+                    return hpo.hp.normal(par, sampler.mean, sampler.sd)
+
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, LogUniform):
+                    if quantize:
+                        return hpo.base.pyll.scope.int(
+                            hpo.hp.qloguniform(
+                                par,
+                                np.log(domain.lower),
+                                np.log(domain.upper),
+                                quantize,
+                            )
+                        )
+                    return hpo.base.pyll.scope.int(
+                        hpo.hp.qloguniform(
+                            par, np.log(domain.lower), np.log(domain.upper - 1), 1.0
+                        )
+                    )
+                elif isinstance(sampler, Uniform):
+                    if quantize:
+                        return hpo.base.pyll.scope.int(
+                            hpo.hp.quniform(
+                                par, domain.lower, domain.upper - 1, quantize
+                            )
+                        )
+                    return hpo.hp.uniformint(par, domain.lower, high=domain.upper - 1)
+            elif isinstance(domain, Categorical):
+                if isinstance(sampler, Uniform):
+                    return hpo.hp.choice(
+                        par,
+                        [
+                            (
+                                HyperOptSearch.convert_search_space(
+                                    category, prefix=par
+                                )
+                                if isinstance(category, dict)
+                                else (
+                                    HyperOptSearch.convert_search_space(
+                                        dict(enumerate(category)), prefix=f"{par}/{i}"
+                                    )
+                                    if isinstance(category, list)
+                                    and len(category) > 0
+                                    and isinstance(category[0], Domain)
+                                    else (
+                                        resolve_value(f"{par}/{i}", category)
+                                        if isinstance(category, Domain)
+                                        else category
+                                    )
+                                )
+                            )
+                            for i, category in enumerate(domain.categories)
+                        ],
+                    )
+
+            raise ValueError(
+                "HyperOpt does not support parameters of type "
+                "`{}` with samplers of type `{}`".format(
+                    type(domain).__name__, type(domain.sampler).__name__
+                )
+            )
+
+        for path, domain in domain_vars:
+            par = "/".join([str(p) for p in ((prefix,) + path if prefix else path)])
+            value = resolve_value(par, domain)
+            assign_value(spec, path, value)
+
+        return spec

python/ray/tune/search/nevergrad/__init__.py

@@ -0,0 +1,3 @@
+from ray.tune.search.nevergrad.nevergrad_search import NevergradSearch
+
+__all__ = ["NevergradSearch"]

python/ray/tune/search/nevergrad/nevergrad_search.py

@@ -0,0 +1,374 @@
+import inspect
+import logging
+import math
+import pickle
+from typing import Dict, List, Optional, Sequence, Type, Union
+
+from ray.tune.result import DEFAULT_METRIC
+from ray.tune.search import (
+    UNDEFINED_METRIC_MODE,
+    UNDEFINED_SEARCH_SPACE,
+    UNRESOLVED_SEARCH_SPACE,
+    Searcher,
+)
+from ray.tune.search.sample import (
+    Categorical,
+    Domain,
+    Float,
+    Integer,
+    LogUniform,
+    Quantized,
+)
+from ray.tune.search.variant_generator import parse_spec_vars
+from ray.tune.utils.util import flatten_dict, unflatten_dict
+
+try:
+    import nevergrad as ng
+    from nevergrad.optimization import Optimizer
+    from nevergrad.optimization.base import ConfiguredOptimizer
+
+    Parameter = ng.p.Parameter
+except ImportError:
+    ng = None
+    Optimizer = None
+    ConfiguredOptimizer = None
+    Parameter = None
+
+logger = logging.getLogger(__name__)
+
+
+class NevergradSearch(Searcher):
+    """Uses Nevergrad to optimize hyperparameters.
+
+    Nevergrad is an open source tool from Facebook for derivative free
+    optimization.  More info can be found at:
+    https://github.com/facebookresearch/nevergrad.
+
+    You will need to install Nevergrad via the following command:
+
+    .. code-block:: bash
+
+        $ pip install nevergrad
+
+    Parameters:
+        optimizer: Optimizer class provided from Nevergrad.
+            See here for available optimizers:
+            https://facebookresearch.github.io/nevergrad/optimizers_ref.html#optimizers
+            This can also be an instance of a `ConfiguredOptimizer`. See the
+            section on configured optimizers in the above link.
+        optimizer_kwargs: Kwargs passed in when instantiating the `optimizer`
+        space: Nevergrad parametrization
+            to be passed to optimizer on instantiation, or list of parameter
+            names if you passed an optimizer object.
+        metric: The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
+        mode: One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute.
+        points_to_evaluate: Initial parameter suggestions to be run
+            first. This is for when you already have some good parameters
+            you want to run first to help the algorithm make better suggestions
+            for future parameters. Needs to be a list of dicts containing the
+            configurations.
+
+    Tune automatically converts search spaces to Nevergrad's format:
+
+    .. code-block:: python
+
+        import nevergrad as ng
+
+        config = {
+            "width": tune.uniform(0, 20),
+            "height": tune.uniform(-100, 100),
+            "activation": tune.choice(["relu", "tanh"])
+        }
+
+        current_best_params = [{
+            "width": 10,
+            "height": 0,
+            "activation": relu",
+        }]
+
+        ng_search = NevergradSearch(
+            optimizer=ng.optimizers.OnePlusOne,
+            metric="mean_loss",
+            mode="min",
+            points_to_evaluate=current_best_params)
+
+        run(my_trainable, config=config, search_alg=ng_search)
+
+    If you would like to pass the search space manually, the code would
+    look like this:
+
+    .. code-block:: python
+
+        import nevergrad as ng
+
+        space = ng.p.Dict(
+            width=ng.p.Scalar(lower=0, upper=20),
+            height=ng.p.Scalar(lower=-100, upper=100),
+            activation=ng.p.Choice(choices=["relu", "tanh"])
+        )
+
+        ng_search = NevergradSearch(
+            optimizer=ng.optimizers.OnePlusOne,
+            space=space,
+            metric="mean_loss",
+            mode="min")
+
+        run(my_trainable, search_alg=ng_search)
+
+    """
+
+    def __init__(
+        self,
+        optimizer: Optional[
+            Union[Optimizer, Type[Optimizer], ConfiguredOptimizer]
+        ] = None,
+        optimizer_kwargs: Optional[Dict] = None,
+        space: Optional[Union[Dict, Parameter]] = None,
+        metric: Optional[str] = None,
+        mode: Optional[str] = None,
+        points_to_evaluate: Optional[List[Dict]] = None,
+    ):
+        assert (
+            ng is not None
+        ), """Nevergrad must be installed!
+            You can install Nevergrad with the command:
+            `pip install nevergrad`."""
+        if mode:
+            assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
+
+        super(NevergradSearch, self).__init__(metric=metric, mode=mode)
+
+        self._space = None
+        self._opt_factory = None
+        self._nevergrad_opt = None
+        self._optimizer_kwargs = optimizer_kwargs or {}
+
+        if points_to_evaluate is None:
+            self._points_to_evaluate = None
+        elif not isinstance(points_to_evaluate, Sequence):
+            raise ValueError(
+                "Invalid object type passed for `points_to_evaluate`: "
+                f"{type(points_to_evaluate)}. "
+                "Please pass a list of points (dictionaries) instead."
+            )
+        else:
+            self._points_to_evaluate = list(points_to_evaluate)
+
+        if isinstance(space, dict) and space:
+            resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
+            if domain_vars or grid_vars:
+                logger.warning(
+                    UNRESOLVED_SEARCH_SPACE.format(par="space", cls=type(self))
+                )
+                space = self.convert_search_space(space)
+
+        if isinstance(optimizer, Optimizer):
+            if space is not None and not isinstance(space, list):
+                raise ValueError(
+                    "If you pass a configured optimizer to Nevergrad, either "
+                    "pass a list of parameter names or None as the `space` "
+                    "parameter."
+                )
+            if self._optimizer_kwargs:
+                raise ValueError(
+                    "If you pass in optimizer kwargs, either pass "
+                    "an `Optimizer` subclass or an instance of "
+                    "`ConfiguredOptimizer`."
+                )
+
+            self._parameters = space
+            self._nevergrad_opt = optimizer
+        elif (
+            inspect.isclass(optimizer) and issubclass(optimizer, Optimizer)
+        ) or isinstance(optimizer, ConfiguredOptimizer):
+            self._opt_factory = optimizer
+            self._parameters = None
+            self._space = space
+        else:
+            raise ValueError(
+                "The `optimizer` argument passed to NevergradSearch must be "
+                "either an `Optimizer` or a `ConfiguredOptimizer`."
+            )
+
+        self._live_trial_mapping = {}
+
+        if self._nevergrad_opt is not None or self._space is not None:
+            self._setup_nevergrad()
+
+    def _setup_nevergrad(self):
+        if self._opt_factory:
+            self._nevergrad_opt = self._opt_factory(
+                self._space, **self._optimizer_kwargs
+            )
+
+        # nevergrad.tell internally minimizes, so "max" => -1
+        if self._mode == "max":
+            self._metric_op = -1.0
+        elif self._mode == "min":
+            self._metric_op = 1.0
+
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
+        if hasattr(self._nevergrad_opt, "instrumentation"):  # added in v0.2.0
+            if self._nevergrad_opt.instrumentation.kwargs:
+                if self._nevergrad_opt.instrumentation.args:
+                    raise ValueError("Instrumented optimizers should use kwargs only")
+                if self._parameters is not None:
+                    raise ValueError(
+                        "Instrumented optimizers should provide "
+                        "None as parameter_names"
+                    )
+            else:
+                if self._parameters is None:
+                    raise ValueError(
+                        "Non-instrumented optimizers should have "
+                        "a list of parameter_names"
+                    )
+                if len(self._nevergrad_opt.instrumentation.args) != 1:
+                    raise ValueError("Instrumented optimizers should use kwargs only")
+        if self._parameters is not None and self._nevergrad_opt.dimension != len(
+            self._parameters
+        ):
+            raise ValueError(
+                "len(parameters_names) must match optimizer "
+                "dimension for non-instrumented optimizers"
+            )
+
+        if self._points_to_evaluate:
+            # Nevergrad is LIFO, so we add the points to evaluate in reverse
+            # order.
+            for i in range(len(self._points_to_evaluate) - 1, -1, -1):
+                self._nevergrad_opt.suggest(self._points_to_evaluate[i])
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        if self._nevergrad_opt is not None or self._space is not None:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        self._setup_nevergrad()
+        return True
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        if not self._nevergrad_opt:
+            raise RuntimeError(
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"
+                )
+            )
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__, metric=self._metric, mode=self._mode
+                )
+            )
+
+        suggested_config = self._nevergrad_opt.ask()
+
+        self._live_trial_mapping[trial_id] = suggested_config
+        # in v0.2.0+, output of ask() is a Candidate,
+        # with fields args and kwargs
+        if not suggested_config.kwargs:
+            if self._parameters:
+                return unflatten_dict(
+                    dict(zip(self._parameters, suggested_config.args[0]))
+                )
+            return unflatten_dict(suggested_config.value)
+        else:
+            return unflatten_dict(suggested_config.kwargs)
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ):
+        """Notification for the completion of trial.
+
+        The result is internally negated when interacting with Nevergrad
+        so that Nevergrad Optimizers can "maximize" this value,
+        as it minimizes on default.
+        """
+        if result:
+            self._process_result(trial_id, result)
+
+        self._live_trial_mapping.pop(trial_id)
+
+    def _process_result(self, trial_id: str, result: Dict):
+        ng_trial_info = self._live_trial_mapping[trial_id]
+        self._nevergrad_opt.tell(ng_trial_info, self._metric_op * result[self._metric])
+
+    def save(self, checkpoint_path: str):
+        save_object = self.__dict__
+        with open(checkpoint_path, "wb") as outputFile:
+            pickle.dump(save_object, outputFile)
+
+    def restore(self, checkpoint_path: str):
+        with open(checkpoint_path, "rb") as inputFile:
+            save_object = pickle.load(inputFile)
+        self.__dict__.update(save_object)
+
+    @staticmethod
+    def convert_search_space(spec: Dict) -> Parameter:
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a Nevergrad search space."
+            )
+
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        def resolve_value(domain: Domain) -> Parameter:
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning(
+                    "Nevergrad does not support quantization. Dropped quantization."
+                )
+                sampler = sampler.get_sampler()
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    return ng.p.Log(
+                        lower=domain.lower, upper=domain.upper, exponent=math.e
+                    )
+                return ng.p.Scalar(lower=domain.lower, upper=domain.upper)
+
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, LogUniform):
+                    return ng.p.Log(
+                        lower=domain.lower,
+                        upper=domain.upper - 1,  # Upper bound exclusive
+                        exponent=math.e,
+                    ).set_integer_casting()
+                return ng.p.Scalar(
+                    lower=domain.lower,
+                    upper=domain.upper - 1,  # Upper bound exclusive
+                ).set_integer_casting()
+
+            elif isinstance(domain, Categorical):
+                return ng.p.Choice(choices=domain.categories)
+
+            raise ValueError(
+                "Nevergrad does not support parameters of type "
+                "`{}` with samplers of type `{}`".format(
+                    type(domain).__name__, type(domain.sampler).__name__
+                )
+            )
+
+        # Parameter name is e.g. "a/b/c" for nested dicts
+        space = {"/".join(path): resolve_value(domain) for path, domain in domain_vars}
+
+        return ng.p.Dict(**space)

python/ray/tune/search/optuna/__init__.py

@@ -0,0 +1,3 @@
+from ray.tune.search.optuna.optuna_search import OptunaSearch
+
+__all__ = ["OptunaSearch"]

python/ray/tune/search/optuna/optuna_search.py

@@ -0,0 +1,731 @@
+import functools
+import inspect
+import logging
+import pickle
+import time
+import warnings
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+from packaging import version
+
+from ray.air.constants import TRAINING_ITERATION
+from ray.tune.result import DEFAULT_METRIC
+from ray.tune.search import (
+    UNDEFINED_METRIC_MODE,
+    UNDEFINED_SEARCH_SPACE,
+    UNRESOLVED_SEARCH_SPACE,
+    Searcher,
+)
+from ray.tune.search.sample import (
+    Categorical,
+    Domain,
+    Float,
+    Integer,
+    LogUniform,
+    Quantized,
+    Uniform,
+)
+from ray.tune.search.variant_generator import parse_spec_vars
+from ray.tune.utils.util import flatten_dict, unflatten_dict, validate_warmstart
+
+try:
+    import optuna as ot
+    from optuna.distributions import BaseDistribution as OptunaDistribution
+    from optuna.samplers import BaseSampler
+    from optuna.storages import BaseStorage
+    from optuna.trial import Trial as OptunaTrial, TrialState as OptunaTrialState
+except ImportError:
+    ot = None
+    OptunaDistribution = None
+    BaseSampler = None
+    BaseStorage = None
+    OptunaTrialState = None
+    OptunaTrial = None
+
+logger = logging.getLogger(__name__)
+
+# print a warning if define by run function takes longer than this to execute
+DEFINE_BY_RUN_WARN_THRESHOLD_S = 1  # 1 is arbitrary
+
+
+class _OptunaTrialSuggestCaptor:
+    """Utility to capture returned values from Optuna's suggest_ methods.
+
+    This will wrap around the ``optuna.Trial` object and decorate all
+    `suggest_` callables with a function capturing the returned value,
+    which will be saved in the ``captured_values`` dict.
+    """
+
+    def __init__(self, ot_trial: OptunaTrial) -> None:
+        self.ot_trial = ot_trial
+        self.captured_values: Dict[str, Any] = {}
+
+    def _get_wrapper(self, func: Callable) -> Callable:
+        sig = inspect.signature(func)
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            # name is always the first arg for suggest_ methods
+            bound = sig.bind_partial(*args, **kwargs)
+            bound.apply_defaults()
+            if "name" not in bound.arguments:
+                raise ValueError("missing required argument: name")
+            name = bound.arguments["name"]
+            ret = func(*args, **kwargs)
+            self.captured_values[name] = ret
+            return ret
+
+        return wrapper
+
+    def __getattr__(self, item_name: str) -> Any:
+        item = getattr(self.ot_trial, item_name)
+        if item_name.startswith("suggest_") and callable(item):
+            return self._get_wrapper(item)
+        return item
+
+
+class OptunaSearch(Searcher):
+    """A wrapper around Optuna to provide trial suggestions.
+
+    `Optuna <https://optuna.org/>`_ is a hyperparameter optimization library.
+    In contrast to other libraries, it employs define-by-run style
+    hyperparameter definitions.
+
+    This Searcher is a thin wrapper around Optuna's search algorithms.
+    You can pass any Optuna sampler, which will be used to generate
+    hyperparameter suggestions.
+
+    Multi-objective optimization is supported.
+
+    .. note::
+        ``OptunaSearch`` requires ``optuna>=3.0``.
+
+    Args:
+        space: Hyperparameter search space definition for
+            Optuna's sampler. This can be either a :class:`dict` with
+            parameter names as keys and ``optuna.distributions`` as values,
+            or a Callable - in which case, it should be a define-by-run
+            function using ``optuna.trial`` to obtain the hyperparameter
+            values. The function should return either a :class:`dict` of
+            constant values with names as keys, or None.
+            For more information, see https://optuna.readthedocs.io\
+/en/stable/tutorial/10_key_features/002_configurations.html.
+
+            .. warning::
+                No actual computation should take place in the define-by-run
+                function. Instead, put the training logic inside the function
+                or class trainable passed to ``tune.Tuner()``.
+
+        metric: The training result objective value attribute. If
+            None but a mode was passed, the anonymous metric ``_metric``
+            will be used per default. Can be a list of metrics for
+            multi-objective optimization.
+        mode: One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute. Can be a list of
+            modes for multi-objective optimization (corresponding to
+            ``metric``).
+        points_to_evaluate: Initial parameter suggestions to be run
+            first. This is for when you already have some good parameters
+            you want to run first to help the algorithm make better suggestions
+            for future parameters. Needs to be a list of dicts containing the
+            configurations.
+        sampler: Optuna sampler used to
+            draw hyperparameter configurations. Defaults to ``TPESampler``,
+            which supports both single- and multi-objective optimization.
+            See https://optuna.readthedocs.io/en/stable/reference/samplers/index.html
+            for available Optuna samplers.
+        study_name: Optuna study name that uniquely identifies the trial
+            results. Defaults to ``"optuna"``.
+        storage: Optuna storage used for storing trial results to
+            storages other than in-memory storage,
+            for instance optuna.storages.RDBStorage.
+        seed: Seed to initialize sampler with. This parameter is only
+            used when ``sampler=None``. In all other cases, the sampler
+            you pass should be initialized with the seed already.
+        evaluated_rewards: If you have previously evaluated the
+            parameters passed in as points_to_evaluate you can avoid
+            re-running those trials by passing in the reward attributes
+            as a list so the optimiser can be told the results without
+            needing to re-compute the trial. Must be the same length as
+            points_to_evaluate.
+
+            .. warning::
+                When using ``evaluated_rewards``, the search space ``space``
+                must be provided as a :class:`dict` with parameter names as
+                keys and ``optuna.distributions`` instances as values. The
+                define-by-run search space definition is not yet supported with
+                this functionality.
+
+    Tune automatically converts search spaces to Optuna's format:
+
+    .. code-block:: python
+
+        from ray.tune.search.optuna import OptunaSearch
+
+        config = {
+            "a": tune.uniform(6, 8)
+            "b": tune.loguniform(1e-4, 1e-2)
+        }
+
+        optuna_search = OptunaSearch(
+            metric="loss",
+            mode="min")
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+            param_space=config,
+        )
+        tuner.fit()
+
+    If you would like to pass the search space manually, the code would
+    look like this:
+
+    .. code-block:: python
+
+        from ray.tune.search.optuna import OptunaSearch
+        import optuna
+
+        space = {
+            "a": optuna.distributions.FloatDistribution(6, 8),
+            "b": optuna.distributions.FloatDistribution(1e-4, 1e-2, log=True),
+        }
+
+        optuna_search = OptunaSearch(
+            space,
+            metric="loss",
+            mode="min")
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+        )
+        tuner.fit()
+
+        # Equivalent Optuna define-by-run function approach:
+
+        def define_search_space(trial: optuna.Trial):
+            trial.suggest_float("a", 6, 8)
+            trial.suggest_float("b", 1e-4, 1e-2, log=True)
+            # training logic goes into trainable, this is just
+            # for search space definition
+
+        optuna_search = OptunaSearch(
+            define_search_space,
+            metric="loss",
+            mode="min")
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+        )
+        tuner.fit()
+
+    Multi-objective optimization is supported:
+
+    .. code-block:: python
+
+        from ray.tune.search.optuna import OptunaSearch
+        import optuna
+
+        space = {
+            "a": optuna.distributions.FloatDistribution(6, 8),
+            "b": optuna.distributions.FloatDistribution(1e-4, 1e-2, log=True),
+        }
+
+        # Note you have to specify metric and mode here instead of
+        # in tune.TuneConfig
+        optuna_search = OptunaSearch(
+            space,
+            metric=["loss1", "loss2"],
+            mode=["min", "max"])
+
+        # Do not specify metric and mode here!
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+        )
+        tuner.fit()
+
+    You can pass configs that will be evaluated first using
+    ``points_to_evaluate``:
+
+    .. code-block:: python
+
+        from ray.tune.search.optuna import OptunaSearch
+        import optuna
+
+        space = {
+            "a": optuna.distributions.FloatDistribution(6, 8),
+            "b": optuna.distributions.FloatDistribution(1e-4, 1e-2, log=True),
+        }
+
+        optuna_search = OptunaSearch(
+            space,
+            points_to_evaluate=[{"a": 6.5, "b": 5e-4}, {"a": 7.5, "b": 1e-3}]
+            metric="loss",
+            mode="min")
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+        )
+        tuner.fit()
+
+    Avoid re-running evaluated trials by passing the rewards together with
+    `points_to_evaluate`:
+
+    .. code-block:: python
+
+        from ray.tune.search.optuna import OptunaSearch
+        import optuna
+
+        space = {
+            "a": optuna.distributions.FloatDistribution(6, 8),
+            "b": optuna.distributions.FloatDistribution(1e-4, 1e-2, log=True),
+        }
+
+        optuna_search = OptunaSearch(
+            space,
+            points_to_evaluate=[{"a": 6.5, "b": 5e-4}, {"a": 7.5, "b": 1e-3}]
+            evaluated_rewards=[0.89, 0.42]
+            metric="loss",
+            mode="min")
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+        )
+        tuner.fit()
+
+    .. versionadded:: 0.8.8
+
+    """
+
+    def __init__(
+        self,
+        space: Optional[
+            Union[
+                Dict[str, "OptunaDistribution"],
+                List[Tuple],
+                Callable[["OptunaTrial"], Optional[Dict[str, Any]]],
+            ]
+        ] = None,
+        metric: Optional[Union[str, List[str]]] = None,
+        mode: Optional[Union[str, List[str]]] = None,
+        points_to_evaluate: Optional[List[Dict]] = None,
+        sampler: Optional["BaseSampler"] = None,
+        study_name: Optional[str] = None,
+        storage: Optional["BaseStorage"] = None,
+        seed: Optional[int] = None,
+        evaluated_rewards: Optional[List] = None,
+    ):
+        assert ot is not None, "Optuna must be installed! Run `pip install optuna`."
+        if version.parse(ot.__version__) < version.parse("3.0.0"):
+            raise ImportError(
+                "`OptunaSearch` requires the `optuna` version to be >= 3.0.0. "
+                'Upgrade with: `pip install -U "optuna>=3.0"`'
+            )
+        super(OptunaSearch, self).__init__(metric=metric, mode=mode)
+
+        if isinstance(space, dict) and space:
+            resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
+            if domain_vars or grid_vars:
+                logger.warning(
+                    UNRESOLVED_SEARCH_SPACE.format(par="space", cls=type(self).__name__)
+                )
+                space = self.convert_search_space(space)
+            else:
+                # Flatten to support nested dicts
+                space = flatten_dict(space, "/")
+
+        self._space = space
+
+        self._points_to_evaluate = points_to_evaluate or []
+        self._evaluated_rewards = evaluated_rewards
+        if study_name:
+            self._study_name = study_name
+        else:
+            self._study_name = "optuna"  # Fixed study name for in-memory storage
+
+        if sampler and seed:
+            logger.warning(
+                "You passed an initialized sampler to `OptunaSearch`. The "
+                "`seed` parameter has to be passed to the sampler directly "
+                "and will be ignored."
+            )
+        elif sampler:
+            assert isinstance(sampler, BaseSampler), (
+                "You can only pass an instance of "
+                "`optuna.samplers.BaseSampler` "
+                "as a sampler to `OptunaSearcher`."
+            )
+
+        self._sampler = sampler
+        self._seed = seed
+
+        if storage:
+            assert isinstance(storage, BaseStorage), (
+                "The `storage` parameter in `OptunaSearcher` must be an instance "
+                "of `optuna.storages.BaseStorage`."
+            )
+        # If storage is not provided, just set self._storage to None
+        # so that the default in-memory storage is used.
+        self._storage = storage
+
+        self._completed_trials = set()
+
+        self._ot_trials = {}
+        self._ot_study = None
+        if self._space:
+            self._setup_study(mode)
+
+    def _setup_study(self, mode: Union[str, list]):
+        if self._metric is None and self._mode:
+            if isinstance(self._mode, list):
+                raise ValueError(
+                    "If ``mode`` is a list (multi-objective optimization "
+                    "case), ``metric`` must be defined."
+                )
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
+        pruner = ot.pruners.NopPruner()
+
+        if self._sampler:
+            sampler = self._sampler
+        else:
+            # TPESampler handles both single- and multi-objective optimization.
+            sampler = ot.samplers.TPESampler(seed=self._seed)
+
+        if isinstance(mode, list):
+            study_direction_args = dict(
+                directions=["minimize" if m == "min" else "maximize" for m in mode],
+            )
+        else:
+            study_direction_args = dict(
+                direction="minimize" if mode == "min" else "maximize",
+            )
+
+        self._ot_study = ot.study.create_study(
+            storage=self._storage,
+            sampler=sampler,
+            pruner=pruner,
+            study_name=self._study_name,
+            load_if_exists=True,
+            **study_direction_args,
+        )
+
+        if self._points_to_evaluate:
+            validate_warmstart(
+                self._space,
+                self._points_to_evaluate,
+                self._evaluated_rewards,
+                validate_point_name_lengths=not callable(self._space),
+            )
+            if self._evaluated_rewards:
+                for point, reward in zip(
+                    self._points_to_evaluate, self._evaluated_rewards
+                ):
+                    self.add_evaluated_point(point, reward)
+            else:
+                for point in self._points_to_evaluate:
+                    self._ot_study.enqueue_trial(point)
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        if self._space:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        self._setup_study(self._mode)
+        return True
+
+    def _suggest_from_define_by_run_func(
+        self,
+        func: Callable[["OptunaTrial"], Optional[Dict[str, Any]]],
+        ot_trial: "OptunaTrial",
+    ) -> Dict:
+        captor = _OptunaTrialSuggestCaptor(ot_trial)
+        time_start = time.time()
+        ret = func(captor)
+        time_taken = time.time() - time_start
+        if time_taken > DEFINE_BY_RUN_WARN_THRESHOLD_S:
+            warnings.warn(
+                "Define-by-run function passed in the `space` argument "
+                f"took {time_taken} seconds to "
+                "run. Ensure that actual computation, training takes "
+                "place inside Tune's train functions or Trainables "
+                "passed to `tune.Tuner()`."
+            )
+        if ret is not None:
+            if not isinstance(ret, dict):
+                raise TypeError(
+                    "The return value of the define-by-run function "
+                    "passed in the `space` argument should be "
+                    "either None or a `dict` with `str` keys. "
+                    f"Got {type(ret)}."
+                )
+            if not all(isinstance(k, str) for k in ret.keys()):
+                raise TypeError(
+                    "At least one of the keys in the dict returned by the "
+                    "define-by-run function passed in the `space` argument "
+                    "was not a `str`."
+                )
+        return {**captor.captured_values, **ret} if ret else captor.captured_values
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        if not self._space:
+            raise RuntimeError(
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"
+                )
+            )
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__, metric=self._metric, mode=self._mode
+                )
+            )
+        if callable(self._space):
+            # Define-by-run case
+            if trial_id not in self._ot_trials:
+                self._ot_trials[trial_id] = self._ot_study.ask()
+
+            ot_trial = self._ot_trials[trial_id]
+
+            params = self._suggest_from_define_by_run_func(self._space, ot_trial)
+        else:
+            # Use Optuna ask interface (since version 2.6.0)
+            if trial_id not in self._ot_trials:
+                self._ot_trials[trial_id] = self._ot_study.ask(
+                    fixed_distributions=self._space
+                )
+            ot_trial = self._ot_trials[trial_id]
+            params = ot_trial.params
+
+        return unflatten_dict(params)
+
+    def on_trial_result(self, trial_id: str, result: Dict):
+        if isinstance(self.metric, list):
+            # Optuna doesn't support incremental results
+            # for multi-objective optimization
+            return
+        if trial_id in self._completed_trials:
+            logger.warning(
+                f"Received additional result for trial {trial_id}, but "
+                f"it already finished. Result: {result}"
+            )
+            return
+        metric = result[self.metric]
+        step = result[TRAINING_ITERATION]
+        ot_trial = self._ot_trials[trial_id]
+        ot_trial.report(metric, step)
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ):
+        if trial_id in self._completed_trials:
+            logger.warning(
+                f"Received additional completion for trial {trial_id}, but "
+                f"it already finished. Result: {result}"
+            )
+            return
+
+        ot_trial = self._ot_trials[trial_id]
+
+        if result:
+            if isinstance(self.metric, list):
+                val = [result.get(metric, None) for metric in self.metric]
+            else:
+                val = result.get(self.metric, None)
+        else:
+            val = None
+        ot_trial_state = OptunaTrialState.COMPLETE
+        if val is None:
+            if error:
+                ot_trial_state = OptunaTrialState.FAIL
+            else:
+                ot_trial_state = OptunaTrialState.PRUNED
+        try:
+            self._ot_study.tell(ot_trial, val, state=ot_trial_state)
+        except Exception as exc:
+            logger.warning(exc)  # E.g. if NaN was reported
+
+        self._completed_trials.add(trial_id)
+
+    def add_evaluated_point(
+        self,
+        parameters: Dict,
+        value: float,
+        error: bool = False,
+        pruned: bool = False,
+        intermediate_values: Optional[List[float]] = None,
+    ):
+        if not self._space:
+            raise RuntimeError(
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"
+                )
+            )
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__, metric=self._metric, mode=self._mode
+                )
+            )
+        if callable(self._space):
+            raise TypeError(
+                "Define-by-run function passed in `space` argument is not "
+                "yet supported when using `evaluated_rewards`. Please provide "
+                "an `OptunaDistribution` dict or pass a Ray Tune "
+                "search space to `tune.Tuner()`."
+            )
+
+        ot_trial_state = OptunaTrialState.COMPLETE
+        if error:
+            ot_trial_state = OptunaTrialState.FAIL
+        elif pruned:
+            ot_trial_state = OptunaTrialState.PRUNED
+
+        if intermediate_values:
+            intermediate_values_dict = dict(enumerate(intermediate_values))
+        else:
+            intermediate_values_dict = None
+
+        # If the trial state is FAILED, the value must be `None` in Optuna==4.1.0
+        # Reference: https://github.com/optuna/optuna/pull/5211
+        # This is a temporary fix for the issue that Optuna enforces the value
+        # to be `None` if the trial state is FAILED.
+        # TODO (hpguo): A better solution may requires us to update the base class
+        # to allow the `value` arg in `add_evaluated_point` being `Optional[float]`.
+        if ot_trial_state == OptunaTrialState.FAIL:
+            value = None
+
+        trial = ot.trial.create_trial(
+            state=ot_trial_state,
+            value=value,
+            params=parameters,
+            distributions=self._space,
+            intermediate_values=intermediate_values_dict,
+        )
+
+        self._ot_study.add_trial(trial)
+
+    def save(self, checkpoint_path: str):
+        save_object = self.__dict__.copy()
+        with open(checkpoint_path, "wb") as outputFile:
+            pickle.dump(save_object, outputFile)
+
+    def restore(self, checkpoint_path: str):
+        with open(checkpoint_path, "rb") as inputFile:
+            save_object = pickle.load(inputFile)
+        if isinstance(save_object, dict):
+            self.__dict__.update(save_object)
+        else:
+            # Backwards compatibility
+            (
+                self._sampler,
+                self._ot_trials,
+                self._ot_study,
+                self._points_to_evaluate,
+                self._evaluated_rewards,
+            ) = save_object
+
+    @staticmethod
+    def convert_search_space(spec: Dict) -> Dict[str, Any]:
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if not domain_vars and not grid_vars:
+            return {}
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to an Optuna search space."
+            )
+
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        def resolve_value(domain: Domain) -> ot.distributions.BaseDistribution:
+            quantize = None
+
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                quantize = sampler.q
+                sampler = sampler.sampler
+                if isinstance(sampler, LogUniform):
+                    logger.warning(
+                        "Optuna does not handle quantization in loguniform "
+                        "sampling. The parameter will be passed but it will "
+                        "probably be ignored."
+                    )
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    if quantize:
+                        logger.warning(
+                            "Optuna does not support both quantization and "
+                            "sampling from LogUniform. Dropped quantization."
+                        )
+                    return ot.distributions.FloatDistribution(
+                        domain.lower, domain.upper, log=True
+                    )
+
+                elif isinstance(sampler, Uniform):
+                    if quantize:
+                        return ot.distributions.FloatDistribution(
+                            domain.lower, domain.upper, step=quantize
+                        )
+                    return ot.distributions.FloatDistribution(
+                        domain.lower, domain.upper
+                    )
+
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, LogUniform):
+                    return ot.distributions.IntDistribution(
+                        domain.lower, domain.upper - 1, step=quantize or 1, log=True
+                    )
+                elif isinstance(sampler, Uniform):
+                    # Upper bound should be inclusive for quantization and
+                    # exclusive otherwise
+                    return ot.distributions.IntDistribution(
+                        domain.lower,
+                        domain.upper - int(bool(not quantize)),
+                        step=quantize or 1,
+                    )
+            elif isinstance(domain, Categorical):
+                if isinstance(sampler, Uniform):
+                    return ot.distributions.CategoricalDistribution(domain.categories)
+
+            raise ValueError(
+                "Optuna search does not support parameters of type "
+                "`{}` with samplers of type `{}`".format(
+                    type(domain).__name__, type(domain.sampler).__name__
+                )
+            )
+
+        # Parameter name is e.g. "a/b/c" for nested dicts
+        values = {"/".join(path): resolve_value(domain) for path, domain in domain_vars}
+
+        return values

python/ray/tune/search/repeater.py

@@ -0,0 +1,199 @@
+import copy
+import logging
+from typing import Dict, List, Optional
+
+import numpy as np
+
+from ray.tune.search.searcher import Searcher
+from ray.tune.search.util import _set_search_properties_backwards_compatible
+from ray.util import PublicAPI
+
+logger = logging.getLogger(__name__)
+
+TRIAL_INDEX = "__trial_index__"
+"""str: A constant value representing the repeat index of the trial."""
+
+
+def _warn_num_samples(searcher: Searcher, num_samples: int):
+    if isinstance(searcher, Repeater) and num_samples % searcher.repeat:
+        logger.warning(
+            "`num_samples` is now expected to be the total number of trials, "
+            "including the repeat trials. For example, set num_samples=15 if "
+            "you intend to obtain 3 search algorithm suggestions and repeat "
+            "each suggestion 5 times. Any leftover trials "
+            "(num_samples mod repeat) will be ignored."
+        )
+
+
+class _TrialGroup:
+    """Internal class for grouping trials of same parameters.
+
+    This is used when repeating trials for reducing training variance.
+
+    Args:
+        primary_trial_id: Trial ID of the "primary trial".
+            This trial is the one that the Searcher is aware of.
+        config: Suggested configuration shared across all trials
+            in the trial group.
+        max_trials: Max number of trials to execute within this group.
+
+    """
+
+    def __init__(self, primary_trial_id: str, config: Dict, max_trials: int = 1):
+        assert isinstance(config, dict), "config is not a dict, got {}".format(config)
+        self.primary_trial_id = primary_trial_id
+        self.config = config
+        self._trials = {primary_trial_id: None}
+        self.max_trials = max_trials
+
+    def add(self, trial_id: str):
+        assert len(self._trials) < self.max_trials
+        self._trials.setdefault(trial_id, None)
+
+    def full(self) -> bool:
+        return len(self._trials) == self.max_trials
+
+    def report(self, trial_id: str, score: float):
+        assert trial_id in self._trials
+        if score is None:
+            raise ValueError("Internal Error: Score cannot be None.")
+        self._trials[trial_id] = score
+
+    def finished_reporting(self) -> bool:
+        return (
+            None not in self._trials.values() and len(self._trials) == self.max_trials
+        )
+
+    def scores(self) -> List[Optional[float]]:
+        return list(self._trials.values())
+
+    def count(self) -> int:
+        return len(self._trials)
+
+
+@PublicAPI
+class Repeater(Searcher):
+    """A wrapper algorithm for repeating trials of same parameters.
+
+    Set tune.TuneConfig(num_samples=...) to be a multiple of `repeat`. For example,
+    set num_samples=15 if you intend to obtain 3 search algorithm suggestions
+    and repeat each suggestion 5 times. Any leftover trials
+    (num_samples mod repeat) will be ignored.
+
+    It is recommended that you do not run an early-stopping TrialScheduler
+    simultaneously.
+
+    Args:
+        searcher: Searcher object that the
+            Repeater will optimize. Note that the Searcher
+            will only see 1 trial among multiple repeated trials.
+            The result/metric passed to the Searcher upon
+            trial completion will be averaged among all repeats.
+        repeat: Number of times to generate a trial with a repeated
+            configuration. Defaults to 1.
+        set_index: Sets a tune.search.repeater.TRIAL_INDEX in
+            Trainable/Function config which corresponds to the index of the
+            repeated trial. This can be used for seeds. Defaults to True.
+
+    Example:
+
+    .. code-block:: python
+
+        from ray.tune.search import Repeater
+
+        search_alg = BayesOptSearch(...)
+        re_search_alg = Repeater(search_alg, repeat=10)
+
+        # Repeat 2 samples 10 times each.
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=re_search_alg,
+                num_samples=20,
+            ),
+        )
+        tuner.fit()
+
+    """
+
+    def __init__(self, searcher: Searcher, repeat: int = 1, set_index: bool = True):
+        self.searcher = searcher
+        self.repeat = repeat
+        self._set_index = set_index
+        self._groups = []
+        self._trial_id_to_group = {}
+        self._current_group = None
+        super(Repeater, self).__init__(
+            metric=self.searcher.metric, mode=self.searcher.mode
+        )
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        if self._current_group is None or self._current_group.full():
+            config = self.searcher.suggest(trial_id)
+            if config is None:
+                return config
+            self._current_group = _TrialGroup(
+                trial_id, copy.deepcopy(config), max_trials=self.repeat
+            )
+            self._groups.append(self._current_group)
+            index_in_group = 0
+        else:
+            index_in_group = self._current_group.count()
+            self._current_group.add(trial_id)
+
+        config = self._current_group.config.copy()
+        if self._set_index:
+            config[TRIAL_INDEX] = index_in_group
+        self._trial_id_to_group[trial_id] = self._current_group
+        return config
+
+    def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, **kwargs):
+        """Stores the score for and keeps track of a completed trial.
+
+        Stores the metric of a trial as nan if any of the following conditions
+        are met:
+
+        1. ``result`` is empty or not provided.
+        2. ``result`` is provided but no metric was provided.
+
+        """
+        if trial_id not in self._trial_id_to_group:
+            logger.error(
+                "Trial {} not in group; cannot report score. "
+                "Seen trials: {}".format(trial_id, list(self._trial_id_to_group))
+            )
+        trial_group = self._trial_id_to_group[trial_id]
+        if not result or self.searcher.metric not in result:
+            score = np.nan
+        else:
+            score = result[self.searcher.metric]
+        trial_group.report(trial_id, score)
+
+        if trial_group.finished_reporting():
+            scores = trial_group.scores()
+            self.searcher.on_trial_complete(
+                trial_group.primary_trial_id,
+                result={self.searcher.metric: np.nanmean(scores)},
+                **kwargs
+            )
+
+    def get_state(self) -> Dict:
+        self_state = self.__dict__.copy()
+        del self_state["searcher"]
+        return self_state
+
+    def set_state(self, state: Dict):
+        self.__dict__.update(state)
+
+    def save(self, checkpoint_path: str):
+        self.searcher.save(checkpoint_path)
+
+    def restore(self, checkpoint_path: str):
+        self.searcher.restore(checkpoint_path)
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        return _set_search_properties_backwards_compatible(
+            self.searcher.set_search_properties, metric, mode, config, **spec
+        )

python/ray/tune/search/sample.py

@@ -0,0 +1,796 @@
+import logging
+import warnings
+from copy import copy
+from inspect import signature
+from math import isclose
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+
+import numpy as np
+
+# Backwards compatibility
+from ray.util.annotations import DeveloperAPI, PublicAPI, RayDeprecationWarning
+
+try:
+    # Added in numpy>=1.17 but we require numpy>=1.16
+    np_random_generator = np.random.Generator
+    LEGACY_RNG = False
+except AttributeError:
+
+    class np_random_generator:
+        pass
+
+    LEGACY_RNG = True
+
+logger = logging.getLogger(__name__)
+
+
+_MISSING = object()  # Sentinel for missing parameters.
+
+
+def _warn_for_base() -> None:
+    warnings.warn(
+        (
+            "The `base` argument is deprecated. "
+            "Please remove it as it is not actually needed in this method."
+        ),
+        RayDeprecationWarning,
+        stacklevel=2,
+    )
+
+
+class _BackwardsCompatibleNumpyRng:
+    """Thin wrapper to ensure backwards compatibility between
+    new and old numpy randomness generators.
+    """
+
+    _rng = None
+
+    def __init__(
+        self,
+        generator_or_seed: Optional[
+            Union["np_random_generator", np.random.RandomState, int]
+        ] = None,
+    ):
+        if generator_or_seed is None or isinstance(
+            generator_or_seed, (np.random.RandomState, np_random_generator)
+        ):
+            self._rng = generator_or_seed
+        elif LEGACY_RNG:
+            self._rng = np.random.RandomState(generator_or_seed)
+        else:
+            self._rng = np.random.default_rng(generator_or_seed)
+
+    @property
+    def legacy_rng(self) -> bool:
+        return not isinstance(self._rng, np_random_generator)
+
+    @property
+    def rng(self):
+        # don't set self._rng to np.random to avoid picking issues
+        return self._rng if self._rng is not None else np.random
+
+    def __getattr__(self, name: str) -> Any:
+        # https://numpy.org/doc/stable/reference/random/new-or-different.html
+        if self.legacy_rng:
+            if name == "integers":
+                name = "randint"
+            elif name == "random":
+                name = "rand"
+        return getattr(self.rng, name)
+
+
+RandomState = Union[
+    None, _BackwardsCompatibleNumpyRng, np_random_generator, np.random.RandomState, int
+]
+
+
+@DeveloperAPI
+class Domain:
+    """Base class to specify a type and valid range to sample parameters from.
+
+    This base class is implemented by parameter spaces, like float ranges
+    (``Float``), integer ranges (``Integer``), or categorical variables
+    (``Categorical``). The ``Domain`` object contains information about
+    valid values (e.g. minimum and maximum values), and exposes methods that
+    allow specification of specific samplers (e.g. ``uniform()`` or
+    ``loguniform()``).
+
+    """
+
+    sampler = None
+    default_sampler_cls = None
+
+    def cast(self, value):
+        """Cast value to domain type"""
+        return value
+
+    def set_sampler(self, sampler, allow_override=False):
+        if self.sampler and not allow_override:
+            raise ValueError(
+                "You can only choose one sampler for parameter "
+                "domains. Existing sampler for parameter {}: "
+                "{}. Tried to add {}".format(
+                    self.__class__.__name__, self.sampler, sampler
+                )
+            )
+        self.sampler = sampler
+
+    def get_sampler(self):
+        sampler = self.sampler
+        if not sampler:
+            sampler = self.default_sampler_cls()
+        return sampler
+
+    def sample(
+        self,
+        config: Optional[Union[List[Dict], Dict]] = None,
+        size: int = 1,
+        random_state: "RandomState" = None,
+    ):
+        if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
+            random_state = _BackwardsCompatibleNumpyRng(random_state)
+        sampler = self.get_sampler()
+        return sampler.sample(self, config=config, size=size, random_state=random_state)
+
+    def is_grid(self):
+        return isinstance(self.sampler, Grid)
+
+    def is_function(self):
+        return False
+
+    def is_valid(self, value: Any):
+        """Returns True if `value` is a valid value in this domain."""
+        raise NotImplementedError
+
+    @property
+    def domain_str(self):
+        return "(unknown)"
+
+
+@DeveloperAPI
+class Sampler:
+    def sample(
+        self,
+        domain: Domain,
+        config: Optional[Union[List[Dict], Dict]] = None,
+        size: int = 1,
+        random_state: "RandomState" = None,
+    ):
+        raise NotImplementedError
+
+
+@DeveloperAPI
+class BaseSampler(Sampler):
+    def __str__(self):
+        return "Base"
+
+
+@DeveloperAPI
+class Uniform(Sampler):
+    def __str__(self):
+        return "Uniform"
+
+
+@DeveloperAPI
+class LogUniform(Sampler):
+    def __init__(self, base: object = _MISSING):
+        if base is not _MISSING:
+            _warn_for_base()
+
+    def __str__(self):
+        return "LogUniform"
+
+
+@DeveloperAPI
+class Normal(Sampler):
+    def __init__(self, mean: float = 0.0, sd: float = 0.0):
+        self.mean = mean
+        self.sd = sd
+
+        assert self.sd > 0, "SD has to be strictly greater than 0"
+
+    def __str__(self):
+        return "Normal"
+
+
+@DeveloperAPI
+class Grid(Sampler):
+    """Dummy sampler used for grid search"""
+
+    def sample(
+        self,
+        domain: Domain,
+        config: Optional[Union[List[Dict], Dict]] = None,
+        size: int = 1,
+        random_state: "RandomState" = None,
+    ):
+        return RuntimeError("Do not call `sample()` on grid.")
+
+
+@DeveloperAPI
+class Float(Domain):
+    class _Uniform(Uniform):
+        def sample(
+            self,
+            domain: "Float",
+            config: Optional[Union[List[Dict], Dict]] = None,
+            size: int = 1,
+            random_state: "RandomState" = None,
+        ):
+            if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
+                random_state = _BackwardsCompatibleNumpyRng(random_state)
+            assert domain.lower > float("-inf"), "Uniform needs a lower bound"
+            assert domain.upper < float("inf"), "Uniform needs a upper bound"
+            items = random_state.uniform(domain.lower, domain.upper, size=size)
+            return items if len(items) > 1 else domain.cast(items[0])
+
+    class _LogUniform(LogUniform):
+        def sample(
+            self,
+            domain: "Float",
+            config: Optional[Union[List[Dict], Dict]] = None,
+            size: int = 1,
+            random_state: "RandomState" = None,
+        ):
+            if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
+                random_state = _BackwardsCompatibleNumpyRng(random_state)
+            assert domain.lower > 0, "LogUniform needs a lower bound greater than 0"
+            assert (
+                0 < domain.upper < float("inf")
+            ), "LogUniform needs a upper bound greater than 0"
+            logmin = np.log(domain.lower)
+            logmax = np.log(domain.upper)
+
+            items = np.exp(random_state.uniform(logmin, logmax, size=size))
+            return items if len(items) > 1 else domain.cast(items[0])
+
+    class _Normal(Normal):
+        def sample(
+            self,
+            domain: "Float",
+            config: Optional[Union[List[Dict], Dict]] = None,
+            size: int = 1,
+            random_state: "RandomState" = None,
+        ):
+            if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
+                random_state = _BackwardsCompatibleNumpyRng(random_state)
+            assert not domain.lower or domain.lower == float(
+                "-inf"
+            ), "Normal sampling does not allow a lower value bound."
+            assert not domain.upper or domain.upper == float(
+                "inf"
+            ), "Normal sampling does not allow a upper value bound."
+            items = random_state.normal(self.mean, self.sd, size=size)
+            return items if len(items) > 1 else domain.cast(items[0])
+
+    default_sampler_cls = _Uniform
+
+    def __init__(self, lower: Optional[float], upper: Optional[float]):
+        # Need to explicitly check for None
+        self.lower = lower if lower is not None else float("-inf")
+        self.upper = upper if upper is not None else float("inf")
+
+    def cast(self, value):
+        return float(value)
+
+    def uniform(self):
+        if not self.lower > float("-inf"):
+            raise ValueError(
+                "Uniform requires a lower bound. Make sure to set the "
+                "`lower` parameter of `Float()`."
+            )
+        if not self.upper < float("inf"):
+            raise ValueError(
+                "Uniform requires a upper bound. Make sure to set the "
+                "`upper` parameter of `Float()`."
+            )
+        new = copy(self)
+        new.set_sampler(self._Uniform())
+        return new
+
+    def loguniform(self, base: object = _MISSING):
+        if base is not _MISSING:
+            _warn_for_base()
+        if not self.lower > 0:
+            raise ValueError(
+                "LogUniform requires a lower bound greater than 0."
+                f"Got: {self.lower}. Did you pass a variable that has "
+                "been log-transformed? If so, pass the non-transformed value "
+                "instead."
+            )
+        if not 0 < self.upper < float("inf"):
+            raise ValueError(
+                "LogUniform requires a upper bound greater than 0. "
+                f"Got: {self.lower}. Did you pass a variable that has "
+                "been log-transformed? If so, pass the non-transformed value "
+                "instead."
+            )
+        new = copy(self)
+        new.set_sampler(self._LogUniform())
+        return new
+
+    def normal(self, mean=0.0, sd=1.0):
+        new = copy(self)
+        new.set_sampler(self._Normal(mean, sd))
+        return new
+
+    def quantized(self, q: float):
+        if self.lower > float("-inf") and not isclose(
+            self.lower / q, round(self.lower / q)
+        ):
+            raise ValueError(
+                f"Your lower variable bound {self.lower} is not divisible by "
+                f"quantization factor {q}."
+            )
+        if self.upper < float("inf") and not isclose(
+            self.upper / q, round(self.upper / q)
+        ):
+            raise ValueError(
+                f"Your upper variable bound {self.upper} is not divisible by "
+                f"quantization factor {q}."
+            )
+
+        new = copy(self)
+        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
+        return new
+
+    def is_valid(self, value: float):
+        return self.lower <= value <= self.upper
+
+    @property
+    def domain_str(self):
+        return f"({self.lower}, {self.upper})"
+
+
+@DeveloperAPI
+class Integer(Domain):
+    class _Uniform(Uniform):
+        def sample(
+            self,
+            domain: "Integer",
+            config: Optional[Union[List[Dict], Dict]] = None,
+            size: int = 1,
+            random_state: "RandomState" = None,
+        ):
+            if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
+                random_state = _BackwardsCompatibleNumpyRng(random_state)
+            items = random_state.integers(domain.lower, domain.upper, size=size)
+            return items if len(items) > 1 else domain.cast(items[0])
+
+    class _LogUniform(LogUniform):
+        def sample(
+            self,
+            domain: "Integer",
+            config: Optional[Union[List[Dict], Dict]] = None,
+            size: int = 1,
+            random_state: "RandomState" = None,
+        ):
+            if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
+                random_state = _BackwardsCompatibleNumpyRng(random_state)
+            assert domain.lower > 0, "LogUniform needs a lower bound greater than 0"
+            assert (
+                0 < domain.upper < float("inf")
+            ), "LogUniform needs a upper bound greater than 0"
+            logmin = np.log(domain.lower)
+            logmax = np.log(domain.upper)
+
+            items = np.exp(random_state.uniform(logmin, logmax, size=size))
+            items = np.floor(items).astype(int)
+            return items if len(items) > 1 else domain.cast(items[0])
+
+    default_sampler_cls = _Uniform
+
+    def __init__(self, lower, upper):
+        self.lower = lower
+        self.upper = upper
+
+    def cast(self, value):
+        return int(value)
+
+    def quantized(self, q: int):
+        new = copy(self)
+        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
+        return new
+
+    def uniform(self):
+        new = copy(self)
+        new.set_sampler(self._Uniform())
+        return new
+
+    def loguniform(self, base: object = _MISSING):
+        if base is not _MISSING:
+            _warn_for_base()
+        if not self.lower > 0:
+            raise ValueError(
+                "LogUniform requires a lower bound greater than 0."
+                f"Got: {self.lower}. Did you pass a variable that has "
+                "been log-transformed? If so, pass the non-transformed value "
+                "instead."
+            )
+        if not 0 < self.upper < float("inf"):
+            raise ValueError(
+                "LogUniform requires a upper bound greater than 0. "
+                f"Got: {self.lower}. Did you pass a variable that has "
+                "been log-transformed? If so, pass the non-transformed value "
+                "instead."
+            )
+        new = copy(self)
+        new.set_sampler(self._LogUniform())
+        return new
+
+    def is_valid(self, value: int):
+        return self.lower <= value <= self.upper
+
+    @property
+    def domain_str(self):
+        return f"({self.lower}, {self.upper})"
+
+
+@DeveloperAPI
+class Categorical(Domain):
+    class _Uniform(Uniform):
+        def sample(
+            self,
+            domain: "Categorical",
+            config: Optional[Union[List[Dict], Dict]] = None,
+            size: int = 1,
+            random_state: "RandomState" = None,
+        ):
+            if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
+                random_state = _BackwardsCompatibleNumpyRng(random_state)
+            # do not use .choice() directly on domain.categories
+            # as that will coerce them to a single dtype
+            indices = random_state.choice(
+                np.arange(0, len(domain.categories)), size=size
+            )
+            items = [domain.categories[index] for index in indices]
+            return items if len(items) > 1 else domain.cast(items[0])
+
+    default_sampler_cls = _Uniform
+
+    def __init__(self, categories: Sequence):
+        self.categories = list(categories)
+
+    def uniform(self):
+        new = copy(self)
+        new.set_sampler(self._Uniform())
+        return new
+
+    def grid(self):
+        new = copy(self)
+        new.set_sampler(Grid())
+        return new
+
+    def __len__(self):
+        return len(self.categories)
+
+    def __getitem__(self, item):
+        return self.categories[item]
+
+    def is_valid(self, value: Any):
+        return value in self.categories
+
+    @property
+    def domain_str(self):
+        return f"{self.categories}"
+
+
+@DeveloperAPI
+class Function(Domain):
+    class _CallSampler(BaseSampler):
+        def __try_fn(self, domain: "Function", config: Dict[str, Any]):
+            try:
+                return domain.func(config)
+            except (AttributeError, KeyError):
+                from ray.tune.search.variant_generator import _UnresolvedAccessGuard
+
+                r = domain.func(_UnresolvedAccessGuard({"config": config}))
+                logger.warning(
+                    "sample_from functions that take a spec dict are "
+                    "deprecated. Please update your function to work with "
+                    "the config dict directly."
+                )
+                return r
+
+        def sample(
+            self,
+            domain: "Function",
+            config: Optional[Union[List[Dict], Dict]] = None,
+            size: int = 1,
+            random_state: "RandomState" = None,
+        ):
+            if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
+                random_state = _BackwardsCompatibleNumpyRng(random_state)
+            if domain.pass_config:
+                items = [
+                    (
+                        self.__try_fn(domain, config[i])
+                        if isinstance(config, list)
+                        else self.__try_fn(domain, config)
+                    )
+                    for i in range(size)
+                ]
+            else:
+                items = [domain.func() for i in range(size)]
+
+            return items if len(items) > 1 else domain.cast(items[0])
+
+    default_sampler_cls = _CallSampler
+
+    def __init__(self, func: Callable):
+        sig = signature(func)
+
+        pass_config = True  # whether we should pass `config` when calling `func`
+        try:
+            sig.bind({})
+        except TypeError:
+            pass_config = False
+
+        if not pass_config:
+            try:
+                sig.bind()
+            except TypeError as exc:
+                raise ValueError(
+                    "The function passed to a `Function` parameter must be "
+                    "callable with either 0 or 1 parameters."
+                ) from exc
+
+        self.pass_config = pass_config
+        self.func = func
+
+    def is_function(self):
+        return True
+
+    def is_valid(self, value: Any):
+        return True  # This is user-defined, so lets not assume anything
+
+    @property
+    def domain_str(self):
+        return f"{self.func}()"
+
+
+@DeveloperAPI
+class Quantized(Sampler):
+    def __init__(self, sampler: Sampler, q: Union[float, int]):
+        self.sampler = sampler
+        self.q = q
+
+        assert self.sampler, "Quantized() expects a sampler instance"
+
+    def get_sampler(self):
+        return self.sampler
+
+    def sample(
+        self,
+        domain: Domain,
+        config: Optional[Union[List[Dict], Dict]] = None,
+        size: int = 1,
+        random_state: "RandomState" = None,
+    ):
+        if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
+            random_state = _BackwardsCompatibleNumpyRng(random_state)
+
+        if self.q == 1:
+            return self.sampler.sample(domain, config, size, random_state=random_state)
+
+        quantized_domain = copy(domain)
+        quantized_domain.lower = np.ceil(domain.lower / self.q) * self.q
+        quantized_domain.upper = np.floor(domain.upper / self.q) * self.q
+        values = self.sampler.sample(
+            quantized_domain, config, size, random_state=random_state
+        )
+        quantized = np.round(np.divide(values, self.q)) * self.q
+
+        if not isinstance(quantized, np.ndarray):
+            return domain.cast(quantized)
+        return list(quantized)
+
+
+@PublicAPI
+def sample_from(func: Callable[[Dict], Any]):
+    """Specify that tune should sample configuration values from this function.
+
+    Use ``sample_from`` to define conditional search spaces, where the value
+    sampled for one parameter depends on the value sampled for another. The
+    callable receives the ``config`` dict, which exposes the values already
+    sampled for the trial.
+
+    Arguments:
+        func: A callable function to draw a sample from.
+
+    Returns:
+        A ``Function`` domain that samples values by calling ``func``.
+
+    Example:
+        >>> import numpy as np
+        >>> from ray import tune
+        >>> # Sample ``b`` from a range that depends on the value of ``a``.
+        >>> param_space = {
+        ...     "a": tune.randint(5, 10),
+        ...     "b": tune.sample_from(
+        ...         lambda config: np.random.randint(0, config["a"])
+        ...     ),
+        ... }
+    """
+    return Function(func)
+
+
+@PublicAPI
+def uniform(lower: float, upper: float):
+    """Sample a float value uniformly between ``lower`` and ``upper``.
+
+    Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
+    ``np.random.uniform(1, 10))``
+
+    """
+    return Float(lower, upper).uniform()
+
+
+@PublicAPI
+def quniform(lower: float, upper: float, q: float):
+    """Sample a quantized float value uniformly between ``lower`` and ``upper``.
+
+    Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
+    ``np.random.uniform(1, 10))``
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+    Quantization makes the upper bound inclusive.
+
+    """
+    return Float(lower, upper).uniform().quantized(q)
+
+
+@PublicAPI
+def loguniform(lower: float, upper: float, base: object = _MISSING):
+    """Sugar for sampling in different orders of magnitude.
+
+    Args:
+        lower: Lower boundary of the output interval (e.g. 1e-4)
+        upper: Upper boundary of the output interval (e.g. 1e-2)
+        base: Deprecated. No longer used.
+
+    Returns:
+        A ``Float`` domain that samples log-uniformly between ``lower`` and ``upper``.
+    """
+    if base is not _MISSING:
+        _warn_for_base()
+    return Float(lower, upper).loguniform()
+
+
+@PublicAPI
+def qloguniform(lower: float, upper: float, q: float, base: object = _MISSING):
+    """Sugar for sampling in different orders of magnitude.
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+
+    Quantization makes the upper bound inclusive.
+
+    Args:
+        lower: Lower boundary of the output interval (e.g. 1e-4)
+        upper: Upper boundary of the output interval (e.g. 1e-2)
+        q: Quantization number. The result will be rounded to an
+            integer increment of this value.
+        base: Deprecated. No longer used.
+
+    Returns:
+        A ``Float`` domain that samples log-uniformly and quantizes by ``q``.
+    """
+    if base is not _MISSING:
+        _warn_for_base()
+    return Float(lower, upper).loguniform().quantized(q)
+
+
+@PublicAPI
+def choice(categories: Sequence):
+    """Sample a categorical value.
+
+    Sampling from ``tune.choice([1, 2])`` is equivalent to sampling from
+    ``np.random.choice([1, 2])``
+
+    """
+    return Categorical(categories).uniform()
+
+
+@PublicAPI
+def randint(lower: int, upper: int):
+    """Sample an integer value uniformly between ``lower`` and ``upper``.
+
+    ``lower`` is inclusive, ``upper`` is exclusive.
+
+    Sampling from ``tune.randint(10)`` is equivalent to sampling from
+    ``np.random.randint(10)``
+
+    .. versionchanged:: 1.5.0
+        When converting Ray Tune configs to searcher-specific search spaces,
+        the lower and upper limits are adjusted to keep compatibility with
+        the bounds stated in the docstring above.
+
+    """
+    return Integer(lower, upper).uniform()
+
+
+@PublicAPI
+def lograndint(lower: int, upper: int, base: object = _MISSING):
+    """Sample an integer value log-uniformly between ``lower`` and ``upper``.
+
+    ``lower`` is inclusive, ``upper`` is exclusive.
+
+    .. versionchanged:: 1.5.0
+        When converting Ray Tune configs to searcher-specific search spaces,
+        the lower and upper limits are adjusted to keep compatibility with
+        the bounds stated in the docstring above.
+
+    """
+    if base is not _MISSING:
+        _warn_for_base()
+    return Integer(lower, upper).loguniform()
+
+
+@PublicAPI
+def qrandint(lower: int, upper: int, q: int = 1):
+    """Sample an integer value uniformly between ``lower`` and ``upper``.
+
+    ``lower`` is inclusive, ``upper`` is also inclusive (!).
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+    Quantization makes the upper bound inclusive.
+
+    .. versionchanged:: 1.5.0
+        When converting Ray Tune configs to searcher-specific search spaces,
+        the lower and upper limits are adjusted to keep compatibility with
+        the bounds stated in the docstring above.
+
+    """
+    return Integer(lower, upper).uniform().quantized(q)
+
+
+@PublicAPI
+def qlograndint(lower: int, upper: int, q: int, base: object = _MISSING):
+    """Sample an integer value log-uniformly between ``lower`` and ``upper``.
+
+    ``lower`` is inclusive, ``upper`` is also inclusive (!).
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+    Quantization makes the upper bound inclusive.
+
+    .. versionchanged:: 1.5.0
+        When converting Ray Tune configs to searcher-specific search spaces,
+        the lower and upper limits are adjusted to keep compatibility with
+        the bounds stated in the docstring above.
+
+    """
+    if base is not _MISSING:
+        _warn_for_base()
+    return Integer(lower, upper).loguniform().quantized(q)
+
+
+@PublicAPI
+def randn(mean: float = 0.0, sd: float = 1.0):
+    """Sample a float value normally with ``mean`` and ``sd``.
+
+    Args:
+        mean: Mean of the normal distribution. Defaults to 0.
+        sd: SD of the normal distribution. Defaults to 1.
+
+    Returns:
+        A ``Float`` domain that samples from a normal distribution.
+    """
+    return Float(None, None).normal(mean, sd)
+
+
+@PublicAPI
+def qrandn(mean: float, sd: float, q: float):
+    """Sample a float value normally with ``mean`` and ``sd``.
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+
+    Args:
+        mean: Mean of the normal distribution.
+        sd: SD of the normal distribution.
+        q: Quantization number. The result will be rounded to an
+            integer increment of this value.
+
+    Returns:
+        A ``Float`` domain that samples normally and quantizes by ``q``.
+    """
+    return Float(None, None).normal(mean, sd).quantized(q)

python/ray/tune/search/search_algorithm.py

@@ -0,0 +1,130 @@
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
+
+from ray.util.annotations import DeveloperAPI
+
+if TYPE_CHECKING:
+    from ray.tune.experiment import Experiment, Trial
+
+
+@DeveloperAPI
+class SearchAlgorithm:
+    """Interface of an event handler API for hyperparameter search.
+
+    Unlike TrialSchedulers, SearchAlgorithms will not have the ability
+    to modify the execution (i.e., stop and pause trials).
+
+    Trials added manually (i.e., via the Client API) will also notify
+    this class upon new events, so custom search algorithms should
+    maintain a list of trials ID generated from this class.
+
+    See also: `ray.tune.search.BasicVariantGenerator`.
+    """
+
+    _finished = False
+
+    _metric = None
+
+    @property
+    def metric(self):
+        return self._metric
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        """Pass search properties to search algorithm.
+
+        This method acts as an alternative to instantiating search algorithms
+        with their own specific search spaces. Instead they can accept a
+        Tune config through this method.
+
+        The search algorithm will usually pass this method to their
+        ``Searcher`` instance.
+
+        Args:
+            metric: Metric to optimize
+            mode: One of ["min", "max"]. Direction to optimize.
+            config: Tune config dict.
+            **spec: Any kwargs for forward compatibility.
+                Info like Experiment.PUBLIC_KEYS is provided through here.
+
+        Returns:
+            True if the search properties were set successfully, False otherwise.
+        """
+        if self._metric and metric:
+            return False
+        if metric:
+            self._metric = metric
+        return True
+
+    @property
+    def total_samples(self):
+        """Get number of total trials to be generated"""
+        return 0
+
+    def add_configurations(
+        self, experiments: Union["Experiment", List["Experiment"], Dict[str, Dict]]
+    ):
+        """Tracks given experiment specifications.
+
+        Arguments:
+            experiments: Experiments to run.
+        """
+        raise NotImplementedError
+
+    def next_trial(self) -> Optional["Trial"]:
+        """Returns single Trial object to be queued into the TrialRunner.
+
+        Returns:
+            trial: Returns a Trial object.
+        """
+        raise NotImplementedError
+
+    def on_trial_result(self, trial_id: str, result: Dict):
+        """Called on each intermediate result returned by a trial.
+
+        This will only be called when the trial is in the RUNNING state.
+
+        Arguments:
+            trial_id: Identifier for the trial.
+            result: Result dictionary.
+        """
+        pass
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ):
+        """Notification for the completion of trial.
+
+        Arguments:
+            trial_id: Identifier for the trial.
+            result: Defaults to None. A dict will
+                be provided with this notification when the trial is in
+                the RUNNING state AND either completes naturally or
+                by manual termination.
+            error: Defaults to False. True if the trial is in
+                the RUNNING state and errors.
+        """
+        pass
+
+    def is_finished(self) -> bool:
+        """Returns True if no trials left to be queued into TrialRunner.
+
+        Can return True before all trials have finished executing.
+        """
+        return self._finished
+
+    def set_finished(self):
+        """Marks the search algorithm as finished."""
+        self._finished = True
+
+    def has_checkpoint(self, dirpath: str) -> bool:
+        """Should return False if restoring is not implemented."""
+        return False
+
+    def save_to_dir(self, dirpath: str, **kwargs):
+        """Saves a search algorithm."""
+        pass
+
+    def restore_from_dir(self, dirpath: str):
+        """Restores a search algorithm along with its wrapped state."""
+        pass

python/ray/tune/search/search_generator.py

@@ -0,0 +1,223 @@
+import copy
+import logging
+from typing import Dict, List, Optional, Union
+
+from ray.tune.error import TuneError
+from ray.tune.experiment import Experiment, Trial, _convert_to_experiment_list
+from ray.tune.experiment.config_parser import _create_trial_from_spec, _make_parser
+from ray.tune.search.search_algorithm import SearchAlgorithm
+from ray.tune.search.searcher import Searcher
+from ray.tune.search.util import _set_search_properties_backwards_compatible
+from ray.tune.search.variant_generator import _resolve_nested_dict, format_vars
+from ray.tune.utils.util import (
+    _atomic_save,
+    _load_newest_checkpoint,
+    flatten_dict,
+    merge_dicts,
+)
+from ray.util.annotations import DeveloperAPI
+
+logger = logging.getLogger(__name__)
+
+
+def _warn_on_repeater(searcher, total_samples):
+    from ray.tune.search.repeater import _warn_num_samples
+
+    _warn_num_samples(searcher, total_samples)
+
+
+@DeveloperAPI
+class SearchGenerator(SearchAlgorithm):
+    """Generates trials to be passed to the TrialRunner.
+
+    Uses the provided ``searcher`` object to generate trials. This class
+    transparently handles repeating trials with score aggregation
+    without embedding logic into the Searcher.
+
+    Args:
+        searcher: Search object that subclasses the Searcher base class. This
+            is then used for generating new hyperparameter samples.
+    """
+
+    CKPT_FILE_TMPL = "search_gen_state-{}.json"
+
+    def __init__(self, searcher: Searcher):
+        assert issubclass(
+            type(searcher), Searcher
+        ), "Searcher should be subclassing Searcher."
+        self.searcher = searcher
+        self._parser = _make_parser()
+        self._experiment = None
+        self._counter = 0  # Keeps track of number of trials created.
+        self._total_samples = 0  # int: total samples to evaluate.
+        self._finished = False
+
+    @property
+    def metric(self):
+        return self.searcher.metric
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        return _set_search_properties_backwards_compatible(
+            self.searcher.set_search_properties, metric, mode, config, **spec
+        )
+
+    @property
+    def total_samples(self):
+        return self._total_samples
+
+    def add_configurations(
+        self, experiments: Union[Experiment, List[Experiment], Dict[str, Dict]]
+    ):
+        """Registers experiment specifications.
+
+        Arguments:
+            experiments: Experiments to run.
+        """
+        assert not self._experiment
+        logger.debug("added configurations")
+        experiment_list = _convert_to_experiment_list(experiments)
+        assert (
+            len(experiment_list) == 1
+        ), "SearchAlgorithms can only support 1 experiment at a time."
+        self._experiment = experiment_list[0]
+        experiment_spec = self._experiment.spec
+        self._total_samples = self._experiment.spec.get("num_samples", 1)
+
+        _warn_on_repeater(self.searcher, self._total_samples)
+        if "run" not in experiment_spec:
+            raise TuneError("Must specify `run` in {}".format(experiment_spec))
+
+    def next_trial(self):
+        """Provides one Trial object to be queued into the TrialRunner.
+
+        Returns:
+            Trial: Returns a single trial.
+        """
+        if not self.is_finished():
+            return self.create_trial_if_possible(self._experiment.spec)
+        return None
+
+    def create_trial_if_possible(self, experiment_spec: Dict) -> Optional[Trial]:
+        logger.debug("creating trial")
+        trial_id = Trial.generate_id()
+        suggested_config = self.searcher.suggest(trial_id)
+        if suggested_config == Searcher.FINISHED:
+            self._finished = True
+            logger.debug("Searcher has finished.")
+            return
+
+        if suggested_config is None:
+            return
+        spec = copy.deepcopy(experiment_spec)
+        spec["config"] = merge_dicts(spec["config"], copy.deepcopy(suggested_config))
+
+        # Create a new trial_id if duplicate trial is created
+        flattened_config = _resolve_nested_dict(spec["config"])
+        self._counter += 1
+        tag = "{0}_{1}".format(str(self._counter), format_vars(flattened_config))
+        trial = _create_trial_from_spec(
+            spec,
+            self._parser,
+            evaluated_params=flatten_dict(suggested_config),
+            experiment_tag=tag,
+            trial_id=trial_id,
+        )
+        return trial
+
+    def on_trial_result(self, trial_id: str, result: Dict):
+        """Notifies the underlying searcher."""
+        self.searcher.on_trial_result(trial_id, result)
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ):
+        self.searcher.on_trial_complete(trial_id=trial_id, result=result, error=error)
+
+    def is_finished(self) -> bool:
+        return self._counter >= self._total_samples or self._finished
+
+    def get_state(self) -> Dict:
+        return {
+            "counter": self._counter,
+            "total_samples": self._total_samples,
+            "finished": self._finished,
+            "experiment": self._experiment,
+        }
+
+    def set_state(self, state: Dict):
+        self._counter = state["counter"]
+        self._total_samples = state["total_samples"]
+        self._finished = state["finished"]
+        self._experiment = state["experiment"]
+
+    def has_checkpoint(self, dirpath: str):
+        return bool(_load_newest_checkpoint(dirpath, self.CKPT_FILE_TMPL.format("*")))
+
+    def save_to_dir(self, dirpath: str, session_str: str):
+        """Saves self + searcher to dir.
+
+        Separates the "searcher" from its wrappers (concurrency, repeating).
+        This allows the user to easily restore a given searcher.
+
+        The save operation is atomic (write/swap).
+
+        Args:
+            dirpath: Filepath to experiment dir.
+            session_str: Unique identifier of the current run
+                session.
+        """
+        searcher = self.searcher
+        search_alg_state = self.get_state()
+        while hasattr(searcher, "searcher"):
+            searcher_name = type(searcher).__name__
+            if searcher_name in search_alg_state:
+                logger.warning(
+                    "There was a duplicate when saving {}. "
+                    "Restore may not work properly.".format(searcher_name)
+                )
+            else:
+                search_alg_state["name:" + searcher_name] = searcher.get_state()
+            searcher = searcher.searcher
+        base_searcher = searcher
+        # We save the base searcher separately for users to easily
+        # separate the searcher.
+        base_searcher.save_to_dir(dirpath, session_str)
+        file_name = self.CKPT_FILE_TMPL.format(session_str)
+        _atomic_save(
+            state=search_alg_state,
+            checkpoint_dir=dirpath,
+            file_name=file_name,
+            tmp_file_name=f"tmp-{file_name}",
+        )
+
+    def restore_from_dir(self, dirpath: str):
+        """Restores self + searcher + search wrappers from dirpath."""
+
+        searcher = self.searcher
+        search_alg_state = _load_newest_checkpoint(
+            dirpath, self.CKPT_FILE_TMPL.format("*")
+        )
+        if not search_alg_state:
+            raise RuntimeError("Unable to find checkpoint in {}.".format(dirpath))
+        while hasattr(searcher, "searcher"):
+            searcher_name = "name:" + type(searcher).__name__
+            if searcher_name not in search_alg_state:
+                names = [
+                    key.split("name:")[1]
+                    for key in search_alg_state
+                    if key.startswith("name:")
+                ]
+                logger.warning(
+                    "{} was not found in the experiment "
+                    "state when restoring. Found {}.".format(searcher_name, names)
+                )
+            else:
+                searcher.set_state(search_alg_state.pop(searcher_name))
+            searcher = searcher.searcher
+        base_searcher = searcher
+
+        logger.debug(f"searching base {base_searcher}")
+        base_searcher.restore_from_dir(dirpath)
+        self.set_state(search_alg_state)

python/ray/tune/search/searcher.py

@@ -0,0 +1,607 @@
+import copy
+import glob
+import logging
+import os
+import uuid
+import warnings
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+from ray.air._internal.usage import tag_searcher
+from ray.tune.search.util import _set_search_properties_backwards_compatible
+from ray.util.annotations import DeveloperAPI, PublicAPI
+from ray.util.debug import log_once
+
+if TYPE_CHECKING:
+    from ray.tune.analysis import ExperimentAnalysis
+    from ray.tune.experiment import Trial
+
+logger = logging.getLogger(__name__)
+
+
+@DeveloperAPI
+class Searcher:
+    """Abstract class for wrapping suggesting algorithms.
+
+    Custom algorithms can extend this class easily by overriding the
+    `suggest` method provide generated parameters for the trials.
+
+    Any subclass that implements ``__init__`` must also call the
+    constructor of this class: ``super(Subclass, self).__init__(...)``.
+
+    To track suggestions and their corresponding evaluations, the method
+    `suggest` will be passed a trial_id, which will be used in
+    subsequent notifications.
+
+    Not all implementations support multi objectives.
+
+    Note to Tune developers: If a new searcher is added, please update
+    `air/_internal/usage.py`.
+
+    Args:
+        metric: The training result objective value attribute. If
+            list then list of training result objective value attributes
+        mode: If string One of {min, max}. If list then
+            list of max and min, determines whether objective is minimizing
+            or maximizing the metric attribute. Must match type of metric.
+
+    .. code-block:: python
+
+        class ExampleSearch(Searcher):
+            def __init__(self, metric="mean_loss", mode="min", **kwargs):
+                super(ExampleSearch, self).__init__(
+                    metric=metric, mode=mode, **kwargs)
+                self.optimizer = Optimizer()
+                self.configurations = {}
+
+            def suggest(self, trial_id):
+                configuration = self.optimizer.query()
+                self.configurations[trial_id] = configuration
+
+            def on_trial_complete(self, trial_id, result, **kwargs):
+                configuration = self.configurations[trial_id]
+                if result and self.metric in result:
+                    self.optimizer.update(configuration, result[self.metric])
+
+        tuner = tune.Tuner(
+            trainable_function,
+            tune_config=tune.TuneConfig(
+                search_alg=ExampleSearch()
+            )
+        )
+        tuner.fit()
+
+
+    """
+
+    FINISHED = "FINISHED"
+    CKPT_FILE_TMPL = "searcher-state-{}.pkl"
+
+    def __init__(
+        self,
+        metric: Optional[str] = None,
+        mode: Optional[str] = None,
+    ):
+        tag_searcher(self)
+        self._metric = metric
+        self._mode = mode
+
+        if not mode or not metric:
+            # Early return to avoid assertions
+            return
+
+        assert isinstance(
+            metric, type(mode)
+        ), "metric and mode must be of the same type"
+        if isinstance(mode, str):
+            assert mode in ["min", "max"], "if `mode` is a str must be 'min' or 'max'!"
+        elif isinstance(mode, list):
+            assert len(mode) == len(metric), "Metric and mode must be the same length"
+            assert all(
+                mod in ["min", "max", "obs"] for mod in mode
+            ), "All of mode must be 'min' or 'max' or 'obs'!"
+        else:
+            raise ValueError("Mode most either be a list or string")
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        """Pass search properties to searcher.
+
+        This method acts as an alternative to instantiating search algorithms
+        with their own specific search spaces. Instead they can accept a
+        Tune config through this method. A searcher should return ``True``
+        if setting the config was successful, or ``False`` if it was
+        unsuccessful, e.g. when the search space has already been set.
+
+        Args:
+            metric: Metric to optimize
+            mode: One of ["min", "max"]. Direction to optimize.
+            config: Tune config dict.
+            **spec: Any kwargs for forward compatibility.
+                Info like Experiment.PUBLIC_KEYS is provided through here.
+
+        Returns:
+            True if the search properties were set successfully, False otherwise.
+        """
+        return False
+
+    def on_trial_result(self, trial_id: str, result: Dict) -> None:
+        """Optional notification for result during training.
+
+        Note that by default, the result dict may include NaNs or
+        may not include the optimization metric. It is up to the
+        subclass implementation to preprocess the result to
+        avoid breaking the optimization process.
+
+        Args:
+            trial_id: A unique string ID for the trial.
+            result: Dictionary of metrics for current training progress.
+                Note that the result dict may include NaNs or
+                may not include the optimization metric. It is up to the
+                subclass implementation to preprocess the result to
+                avoid breaking the optimization process.
+        """
+        pass
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ) -> None:
+        """Notification for the completion of trial.
+
+        Typically, this method is used for notifying the underlying
+        optimizer of the result.
+
+        Args:
+            trial_id: A unique string ID for the trial.
+            result: Dictionary of metrics for current training progress.
+                Note that the result dict may include NaNs or
+                may not include the optimization metric. It is up to the
+                subclass implementation to preprocess the result to
+                avoid breaking the optimization process. Upon errors, this
+                may also be None.
+            error: True if the training process raised an error.
+
+        """
+        raise NotImplementedError
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        """Queries the algorithm to retrieve the next set of parameters.
+
+        Arguments:
+            trial_id: Trial ID used for subsequent notifications.
+
+        Returns:
+            dict | FINISHED | None: Configuration for a trial, if possible.
+                If FINISHED is returned, Tune will be notified that
+                no more suggestions/configurations will be provided.
+                If None is returned, Tune will skip the querying of the
+                searcher for this step.
+
+        """
+        raise NotImplementedError
+
+    def add_evaluated_point(
+        self,
+        parameters: Dict,
+        value: float,
+        error: bool = False,
+        pruned: bool = False,
+        intermediate_values: Optional[List[float]] = None,
+    ):
+        """Pass results from a point that has been evaluated separately.
+
+        This method allows for information from outside the
+        suggest - on_trial_complete loop to be passed to the search
+        algorithm.
+        This functionality depends on the underlying search algorithm
+        and may not be always available.
+
+        Args:
+            parameters: Parameters used for the trial.
+            value: Metric value obtained in the trial.
+            error: True if the training process raised an error.
+            pruned: True if trial was pruned.
+            intermediate_values: List of metric values for
+                intermediate iterations of the result. None if not
+                applicable.
+
+        """
+        raise NotImplementedError
+
+    def add_evaluated_trials(
+        self,
+        trials_or_analysis: Union["Trial", List["Trial"], "ExperimentAnalysis"],
+        metric: str,
+    ):
+        """Pass results from trials that have been evaluated separately.
+
+        This method allows for information from outside the
+        suggest - on_trial_complete loop to be passed to the search
+        algorithm.
+        This functionality depends on the underlying search algorithm
+        and may not be always available (same as ``add_evaluated_point``.)
+
+        Args:
+            trials_or_analysis: Trials to pass results form to the searcher.
+            metric: Metric name reported by trials used for
+                determining the objective value.
+
+        """
+        if self.add_evaluated_point == Searcher.add_evaluated_point:
+            raise NotImplementedError
+
+        # lazy imports to avoid circular dependencies
+        from ray.tune.analysis import ExperimentAnalysis
+        from ray.tune.experiment import Trial
+        from ray.tune.result import DONE
+
+        if isinstance(trials_or_analysis, (list, tuple)):
+            trials = trials_or_analysis
+        elif isinstance(trials_or_analysis, Trial):
+            trials = [trials_or_analysis]
+        elif isinstance(trials_or_analysis, ExperimentAnalysis):
+            trials = trials_or_analysis.trials
+        else:
+            raise NotImplementedError(
+                "Expected input to be a `Trial`, a list of `Trial`s, or "
+                f"`ExperimentAnalysis`, got: {trials_or_analysis}"
+            )
+
+        any_trial_had_metric = False
+
+        def trial_to_points(trial: Trial) -> Dict[str, Any]:
+            nonlocal any_trial_had_metric
+            has_trial_been_pruned = (
+                trial.status == Trial.TERMINATED
+                and not trial.last_result.get(DONE, False)
+            )
+            has_trial_finished = (
+                trial.status == Trial.TERMINATED and trial.last_result.get(DONE, False)
+            )
+            if not any_trial_had_metric:
+                any_trial_had_metric = (
+                    metric in trial.last_result and has_trial_finished
+                )
+            if Trial.TERMINATED and metric not in trial.last_result:
+                return None
+            return dict(
+                parameters=trial.config,
+                value=trial.last_result.get(metric, None),
+                error=trial.status == Trial.ERROR,
+                pruned=has_trial_been_pruned,
+                intermediate_values=None,  # we do not save those
+            )
+
+        for trial in trials:
+            kwargs = trial_to_points(trial)
+            if kwargs:
+                self.add_evaluated_point(**kwargs)
+
+        if not any_trial_had_metric:
+            warnings.warn(
+                "No completed trial returned the specified metric. "
+                "Make sure the name you have passed is correct. "
+            )
+
+    def save(self, checkpoint_path: str):
+        """Save state to path for this search algorithm.
+
+        Args:
+            checkpoint_path: File where the search algorithm
+                state is saved. This path should be used later when
+                restoring from file.
+
+        Example:
+
+        .. code-block:: python
+
+            search_alg = Searcher(...)
+
+            tuner = tune.Tuner(
+                cost,
+                tune_config=tune.TuneConfig(
+                    search_alg=search_alg,
+                    num_samples=5
+                ),
+                param_space=config
+            )
+            results = tuner.fit()
+
+            search_alg.save("./my_favorite_path.pkl")
+
+        .. versionchanged:: 0.8.7
+            Save is automatically called by `Tuner().fit()`. You can use
+            `Tuner().restore()` to restore from an experiment directory
+            such as `~/ray_results/trainable`.
+
+        """
+        raise NotImplementedError
+
+    def restore(self, checkpoint_path: str):
+        """Restore state for this search algorithm
+
+
+        Args:
+            checkpoint_path: File where the search algorithm
+                state is saved. This path should be the same
+                as the one provided to "save".
+
+        Example:
+
+        .. code-block:: python
+
+            search_alg.save("./my_favorite_path.pkl")
+
+            search_alg2 = Searcher(...)
+            search_alg2 = ConcurrencyLimiter(search_alg2, 1)
+            search_alg2.restore(checkpoint_path)
+            tuner = tune.Tuner(
+                cost,
+                tune_config=tune.TuneConfig(
+                    search_alg=search_alg2,
+                    num_samples=5
+                ),
+            )
+            tuner.fit()
+
+        """
+        raise NotImplementedError
+
+    def set_max_concurrency(self, max_concurrent: int) -> bool:
+        """Set max concurrent trials this searcher can run.
+
+        This method will be called on the wrapped searcher by the
+        ``ConcurrencyLimiter``. It is intended to allow for searchers
+        which have custom, internal logic handling max concurrent trials
+        to inherit the value passed to ``ConcurrencyLimiter``.
+
+        If this method returns False, it signifies that no special
+        logic for handling this case is present in the searcher.
+
+        Args:
+            max_concurrent: Number of maximum concurrent trials.
+
+        Returns:
+            True if the searcher handles max concurrency internally,
+            False otherwise.
+        """
+        return False
+
+    def get_state(self) -> Dict:
+        raise NotImplementedError
+
+    def set_state(self, state: Dict):
+        raise NotImplementedError
+
+    def save_to_dir(self, checkpoint_dir: str, session_str: str = "default"):
+        """Automatically saves the given searcher to the checkpoint_dir.
+
+        This is automatically used by Tuner().fit() during a Tune job.
+
+        Args:
+            checkpoint_dir: Filepath to experiment dir.
+            session_str: Unique identifier of the current run
+                session.
+        """
+        file_name = self.CKPT_FILE_TMPL.format(session_str)
+        tmp_file_name = f".{str(uuid.uuid4())}-tmp-{file_name}"
+        tmp_search_ckpt_path = os.path.join(checkpoint_dir, tmp_file_name)
+        success = True
+        try:
+            self.save(tmp_search_ckpt_path)
+        except NotImplementedError:
+            if log_once("suggest:save_to_dir"):
+                logger.warning("save not implemented for Searcher. Skipping save.")
+            success = False
+
+        if success and os.path.exists(tmp_search_ckpt_path):
+            os.replace(
+                tmp_search_ckpt_path,
+                os.path.join(checkpoint_dir, file_name),
+            )
+
+    def restore_from_dir(self, checkpoint_dir: str):
+        """Restores the state of a searcher from a given checkpoint_dir.
+
+        Typically, you should use this function to restore from an
+        experiment directory such as `~/ray_results/trainable`.
+
+        .. code-block:: python
+
+            tuner = tune.Tuner(
+                cost,
+                run_config=tune.RunConfig(
+                    name=self.experiment_name,
+                    storage_path="~/my_results",
+                ),
+                tune_config=tune.TuneConfig(
+                    search_alg=search_alg,
+                    num_samples=5
+                ),
+                param_space=config
+            )
+            tuner.fit()
+
+            search_alg2 = Searcher()
+            search_alg2.restore_from_dir(
+                os.path.join("~/my_results", self.experiment_name)
+        """
+
+        pattern = self.CKPT_FILE_TMPL.format("*")
+        full_paths = glob.glob(os.path.join(checkpoint_dir, pattern))
+        if not full_paths:
+            raise RuntimeError(
+                "Searcher unable to find checkpoint in {}".format(checkpoint_dir)
+            )  # TODO
+        most_recent_checkpoint = max(full_paths)
+        self.restore(most_recent_checkpoint)
+
+    @property
+    def metric(self) -> str:
+        """The training result objective value attribute."""
+        return self._metric
+
+    @property
+    def mode(self) -> str:
+        """Specifies if minimizing or maximizing the metric."""
+        return self._mode
+
+
+@PublicAPI
+class ConcurrencyLimiter(Searcher):
+    """A wrapper algorithm for limiting the number of concurrent trials.
+
+    Certain Searchers have their own internal logic for limiting
+    the number of concurrent trials. If such a Searcher is passed to a
+    ``ConcurrencyLimiter``, the ``max_concurrent`` of the
+    ``ConcurrencyLimiter`` will override the ``max_concurrent`` value
+    of the Searcher. The ``ConcurrencyLimiter`` will then let the
+    Searcher's internal logic take over.
+
+    Args:
+        searcher: Searcher object that the
+            ConcurrencyLimiter will manage.
+        max_concurrent: Maximum concurrent samples from the underlying
+            searcher.
+        batch: Whether to wait for all concurrent samples
+            to finish before updating the underlying searcher.
+
+    Example:
+
+    .. code-block:: python
+
+        from ray.tune.search import ConcurrencyLimiter
+        search_alg = HyperOptSearch(metric="accuracy")
+        search_alg = ConcurrencyLimiter(search_alg, max_concurrent=2)
+        tuner = tune.Tuner(
+            trainable_function,
+            tune_config=tune.TuneConfig(
+                search_alg=search_alg
+            ),
+        )
+        tuner.fit()
+
+    """
+
+    def __init__(self, searcher: Searcher, max_concurrent: int, batch: bool = False):
+        assert isinstance(max_concurrent, int) and max_concurrent > 0
+        self.searcher = searcher
+        self.max_concurrent = max_concurrent
+        self.batch = batch
+        self.live_trials = set()
+        self.num_unfinished_live_trials = 0
+        self.cached_results = {}
+        self._limit_concurrency = True
+
+        if not isinstance(searcher, Searcher):
+            raise RuntimeError(
+                f"The `ConcurrencyLimiter` only works with `Searcher` "
+                f"objects (got {type(searcher)}). Please try to pass "
+                f"`max_concurrent` to the search generator directly."
+            )
+
+        self._set_searcher_max_concurrency()
+
+        super(ConcurrencyLimiter, self).__init__(
+            metric=self.searcher.metric, mode=self.searcher.mode
+        )
+
+    def _set_searcher_max_concurrency(self):
+        # If the searcher has special logic for handling max concurrency,
+        # we do not do anything inside the ConcurrencyLimiter
+        self._limit_concurrency = not self.searcher.set_max_concurrency(
+            self.max_concurrent
+        )
+
+    def set_max_concurrency(self, max_concurrent: int) -> bool:
+        # Determine if this behavior is acceptable, or if it should
+        # raise an exception.
+        self.max_concurrent = max_concurrent
+        return True
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        self._set_searcher_max_concurrency()
+        return _set_search_properties_backwards_compatible(
+            self.searcher.set_search_properties, metric, mode, config, **spec
+        )
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        if not self._limit_concurrency:
+            return self.searcher.suggest(trial_id)
+
+        assert (
+            trial_id not in self.live_trials
+        ), f"Trial ID {trial_id} must be unique: already found in set."
+        if len(self.live_trials) >= self.max_concurrent:
+            logger.debug(
+                f"Not providing a suggestion for {trial_id} due to "
+                "concurrency limit: %s/%s.",
+                len(self.live_trials),
+                self.max_concurrent,
+            )
+            return
+
+        suggestion = self.searcher.suggest(trial_id)
+        if suggestion not in (None, Searcher.FINISHED):
+            self.live_trials.add(trial_id)
+            self.num_unfinished_live_trials += 1
+        return suggestion
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ):
+        if not self._limit_concurrency:
+            return self.searcher.on_trial_complete(trial_id, result=result, error=error)
+
+        if trial_id not in self.live_trials:
+            return
+        elif self.batch:
+            self.cached_results[trial_id] = (result, error)
+            self.num_unfinished_live_trials -= 1
+            if self.num_unfinished_live_trials <= 0:
+                # Update the underlying searcher once the
+                # full batch is completed.
+                for trial_id, (result, error) in self.cached_results.items():
+                    self.searcher.on_trial_complete(
+                        trial_id, result=result, error=error
+                    )
+                    self.live_trials.remove(trial_id)
+                self.cached_results = {}
+                self.num_unfinished_live_trials = 0
+            else:
+                return
+        else:
+            self.searcher.on_trial_complete(trial_id, result=result, error=error)
+            self.live_trials.remove(trial_id)
+            self.num_unfinished_live_trials -= 1
+
+    def on_trial_result(self, trial_id: str, result: Dict) -> None:
+        self.searcher.on_trial_result(trial_id, result)
+
+    def add_evaluated_point(
+        self,
+        parameters: Dict,
+        value: float,
+        error: bool = False,
+        pruned: bool = False,
+        intermediate_values: Optional[List[float]] = None,
+    ):
+        return self.searcher.add_evaluated_point(
+            parameters, value, error, pruned, intermediate_values
+        )
+
+    def get_state(self) -> Dict:
+        state = self.__dict__.copy()
+        del state["searcher"]
+        return copy.deepcopy(state)
+
+    def set_state(self, state: Dict):
+        self.__dict__.update(state)
+
+    def save(self, checkpoint_path: str):
+        self.searcher.save(checkpoint_path)
+
+    def restore(self, checkpoint_path: str):
+        self.searcher.restore(checkpoint_path)

python/ray/tune/search/util.py

@@ -0,0 +1,29 @@
+import logging
+from typing import Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def _set_search_properties_backwards_compatible(
+    set_search_properties_func,
+    metric: Optional[str],
+    mode: Optional[str],
+    config: Dict,
+    **spec
+) -> bool:
+    """Wraps around set_search_properties() so that it is backward compatible.
+
+    Also outputs a warning to encourage custom searchers to be updated.
+    """
+    try:
+        return set_search_properties_func(metric, mode, config, **spec)
+    except TypeError as e:
+        if "set_search_properties() got an unexpected keyword argument" in str(e):
+            logger.warning(
+                "Please update custom Searcher to take in function signature "
+                "as ``def set_search_properties(metric, mode, config, "
+                "**spec) -> bool``."
+            )
+            return set_search_properties_func(metric, mode, config)
+        else:
+            raise e

python/ray/tune/search/variant_generator.py

@@ -0,0 +1,533 @@
+import copy
+import logging
+import random
+import re
+from collections.abc import Mapping
+from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple
+
+import numpy
+
+from ray.tune.search.sample import Categorical, Domain, Function, RandomState
+from ray.util.annotations import DeveloperAPI, PublicAPI
+
+logger = logging.getLogger(__name__)
+
+
+@DeveloperAPI
+def generate_variants(
+    unresolved_spec: Dict,
+    constant_grid_search: bool = False,
+    random_state: "RandomState" = None,
+) -> Generator[Tuple[Dict, Dict], None, None]:
+    """Generates variants from a spec (dict) with unresolved values.
+
+    There are two types of unresolved values:
+
+        Grid search: These define a grid search over values. For example, the
+        following grid search values in a spec will produce six distinct
+        variants in combination:
+
+            "activation": grid_search(["relu", "tanh"])
+            "learning_rate": grid_search([1e-3, 1e-4, 1e-5])
+
+        Lambda functions: These are evaluated to produce a concrete value, and
+        can express dependencies or conditional distributions between values.
+        They can also be used to express random search (e.g., by calling
+        into the `random` or `np` module).
+
+            "cpu": lambda spec: spec.config.num_workers
+            "batch_size": lambda spec: random.uniform(1, 1000)
+
+    Finally, to support defining specs in plain JSON / YAML, grid search
+    and lambda functions can also be defined alternatively as follows:
+
+        "activation": {"grid_search": ["relu", "tanh"]}
+        "cpu": {"eval": "spec.config.num_workers"}
+
+    Use `format_vars` to format the returned dict of hyperparameters.
+
+    Args:
+        unresolved_spec: Experiment spec containing unresolved variants.
+        constant_grid_search: If True, sample random variables once before
+            iterating over grid variants; if False, resample for each grid variant.
+        random_state: Seed or numpy random generator used to draw random samples.
+
+    Yields:
+        Tuple[Dict, Dict]: ``(resolved_vars, spec)`` pairs, where ``resolved_vars``
+            is a dict of resolved variables and ``spec`` is the fully resolved spec.
+    """
+    for resolved_vars, spec in _generate_variants_internal(
+        unresolved_spec,
+        constant_grid_search=constant_grid_search,
+        random_state=random_state,
+    ):
+        assert not _unresolved_values(spec)
+        yield resolved_vars, spec
+
+
+@PublicAPI(stability="beta")
+def grid_search(values: Iterable) -> Dict[str, Iterable]:
+    """Specify a grid of values to search over.
+
+    Values specified in a grid search are guaranteed to be sampled.
+
+    If multiple grid search variables are defined, they are combined with the
+    combinatorial product. This means every possible combination of values will
+    be sampled.
+
+    Example:
+
+        >>> from ray import tune
+        >>> param_space={
+        ...   "x": tune.grid_search([10, 20]),
+        ...   "y": tune.grid_search(["a", "b", "c"])
+        ... }
+
+    This will create a grid of 6 samples:
+    ``{"x": 10, "y": "a"}``, ``{"x": 10, "y": "b"}``, etc.
+
+    When specifying ``num_samples`` in the
+    :class:`TuneConfig <ray.tune.tune_config.TuneConfig>`, this will specify
+    the number of random samples per grid search combination.
+
+    For instance, in the example above, if ``num_samples=4``,
+    a total of 24 trials will be started -
+    4 trials for each of the 6 grid search combinations.
+
+    Args:
+        values: An iterable whose parameters will be used for creating a trial grid.
+
+    Returns:
+        A dict in the form ``{"grid_search": values}`` understood by Tune's
+        variant generator.
+    """
+    return {"grid_search": values}
+
+
+_STANDARD_IMPORTS = {
+    "random": random,
+    "np": numpy,
+}
+
+_MAX_RESOLUTION_PASSES = 20
+
+
+def _resolve_nested_dict(nested_dict: Dict) -> Dict[Tuple, Any]:
+    """Flattens a nested dict by joining keys into tuple of paths.
+
+    Can then be passed into `format_vars`.
+    """
+    res = {}
+    for k, v in nested_dict.items():
+        if isinstance(v, dict):
+            for k_, v_ in _resolve_nested_dict(v).items():
+                res[(k,) + k_] = v_
+        else:
+            res[(k,)] = v
+    return res
+
+
+@DeveloperAPI
+def format_vars(resolved_vars: Dict) -> str:
+    """Format variables to be used as experiment tags.
+
+    Experiment tags are used in directory names, so this method makes sure
+    the resulting tags can be legally used in directory names on all systems.
+
+    The input to this function is a dict of the form
+    ``{("nested", "config", "path"): "value"}``. The output will be a comma
+    separated string of the form ``last_key=value``, so in this example
+    ``path=value``.
+
+    Note that the sanitizing implies that empty strings are possible return
+    values. This is expected and acceptable, as it is not a common case and
+    the resulting directory names will still be valid.
+
+    Args:
+        resolved_vars: Dictionary mapping from config path tuples to a value.
+
+    Returns:
+        Comma-separated key=value string.
+    """
+    vars = resolved_vars.copy()
+    # TrialRunner already has these in the experiment_tag
+    for v in ["run", "env", "resources_per_trial"]:
+        vars.pop(v, None)
+
+    return ",".join(
+        f"{_clean_value(k[-1])}={_clean_value(v)}" for k, v in sorted(vars.items())
+    )
+
+
+def _flatten_resolved_vars(resolved_vars: Dict) -> Dict:
+    """Formats the resolved variable dict into a mapping of (str -> value)."""
+    flattened_resolved_vars_dict = {}
+    for pieces, value in resolved_vars.items():
+        if pieces[0] == "config":
+            pieces = pieces[1:]
+        pieces = [str(piece) for piece in pieces]
+        flattened_resolved_vars_dict["/".join(pieces)] = value
+    return flattened_resolved_vars_dict
+
+
+def _clean_value(value: Any) -> str:
+    """Format floats and replace invalid string characters with ``_``."""
+    if isinstance(value, float):
+        return f"{value:.4f}"
+    else:
+        # Define an invalid alphabet, which is the inverse of the
+        # stated regex characters
+        invalid_alphabet = r"[^a-zA-Z0-9_-]+"
+        return re.sub(invalid_alphabet, "_", str(value)).strip("_")
+
+
+@DeveloperAPI
+def parse_spec_vars(
+    spec: Dict,
+) -> Tuple[List[Tuple[Tuple, Any]], List[Tuple[Tuple, Any]], List[Tuple[Tuple, Any]]]:
+    resolved, unresolved = _split_resolved_unresolved_values(spec)
+    resolved_vars = list(resolved.items())
+
+    if not unresolved:
+        return resolved_vars, [], []
+
+    grid_vars = []
+    domain_vars = []
+    for path, value in unresolved.items():
+        if value.is_grid():
+            grid_vars.append((path, value))
+        else:
+            domain_vars.append((path, value))
+    grid_vars.sort()
+
+    return resolved_vars, domain_vars, grid_vars
+
+
+def _count_spec_samples(spec: Dict, num_samples=1) -> int:
+    """Count samples for a specific spec"""
+    _, domain_vars, grid_vars = parse_spec_vars(spec)
+    grid_count = 1
+    for path, domain in grid_vars:
+        grid_count *= len(domain.categories)
+    return num_samples * grid_count
+
+
+def _count_variants(spec: Dict, presets: Optional[List[Dict]] = None) -> int:
+    # Helper function: Deep update dictionary
+    def deep_update(d, u):
+        for k, v in u.items():
+            if isinstance(v, Mapping):
+                d[k] = deep_update(d.get(k, {}), v)
+            else:
+                d[k] = v
+        return d
+
+    total_samples = 0
+    total_num_samples = spec.get("num_samples", 1)
+    # For each preset, overwrite the spec and count the samples generated
+    # for this preset
+    for preset in presets:
+        preset_spec = copy.deepcopy(spec)
+        deep_update(preset_spec["config"], preset)
+        total_samples += _count_spec_samples(preset_spec, 1)
+        total_num_samples -= 1
+
+    # Add the remaining samples
+    if total_num_samples > 0:
+        total_samples += _count_spec_samples(spec, total_num_samples)
+    return total_samples
+
+
+def _generate_variants_internal(
+    spec: Dict, constant_grid_search: bool = False, random_state: "RandomState" = None
+) -> Tuple[Dict, Dict]:
+    spec = copy.deepcopy(spec)
+    _, domain_vars, grid_vars = parse_spec_vars(spec)
+
+    if not domain_vars and not grid_vars:
+        yield {}, spec
+        return
+
+    # Variables to resolve
+    to_resolve = domain_vars
+
+    all_resolved = True
+    if constant_grid_search:
+        # In this path, we first sample random variables and keep them constant
+        # for grid search.
+        # `_resolve_domain_vars` will alter `spec` directly
+        all_resolved, resolved_vars = _resolve_domain_vars(
+            spec, domain_vars, allow_fail=True, random_state=random_state
+        )
+        if not all_resolved:
+            # Not all variables have been resolved, but remove those that have
+            # from the `to_resolve` list.
+            to_resolve = [(r, d) for r, d in to_resolve if r not in resolved_vars]
+    grid_search = _grid_search_generator(spec, grid_vars)
+    for resolved_spec in grid_search:
+        if not constant_grid_search or not all_resolved:
+            # In this path, we sample the remaining random variables
+            _, resolved_vars = _resolve_domain_vars(
+                resolved_spec, to_resolve, random_state=random_state
+            )
+
+        for resolved, spec in _generate_variants_internal(
+            resolved_spec,
+            constant_grid_search=constant_grid_search,
+            random_state=random_state,
+        ):
+            for path, value in grid_vars:
+                resolved_vars[path] = _get_value(spec, path)
+            for k, v in resolved.items():
+                if (
+                    k in resolved_vars
+                    and v != resolved_vars[k]
+                    and _is_resolved(resolved_vars[k])
+                ):
+                    raise ValueError(
+                        "The variable `{}` could not be unambiguously "
+                        "resolved to a single value. Consider simplifying "
+                        "your configuration.".format(k)
+                    )
+                resolved_vars[k] = v
+            yield resolved_vars, spec
+
+
+def _get_preset_variants(
+    spec: Dict,
+    config: Dict,
+    constant_grid_search: bool = False,
+    random_state: "RandomState" = None,
+):
+    """Get variants according to a spec, initialized with a config.
+
+    Variables from the spec are overwritten by the variables in the config.
+    Thus, we may end up with less sampled parameters.
+
+    This function also checks if values used to overwrite search space
+    parameters are valid, and logs a warning if not.
+    """
+    spec = copy.deepcopy(spec)
+
+    resolved, _, _ = parse_spec_vars(config)
+
+    for path, val in resolved:
+        try:
+            domain = _get_value(spec["config"], path)
+            if isinstance(domain, dict):
+                if "grid_search" in domain:
+                    domain = Categorical(domain["grid_search"])
+                else:
+                    # If users want to overwrite an entire subdict,
+                    # let them do it.
+                    domain = None
+        except IndexError as exc:
+            raise ValueError(
+                f"Pre-set config key `{'/'.join(path)}` does not correspond "
+                f"to a valid key in the search space definition. Please add "
+                f"this path to the `param_space` variable passed to `tune.Tuner()`."
+            ) from exc
+
+        if domain:
+            if isinstance(domain, Domain):
+                if not domain.is_valid(val):
+                    logger.warning(
+                        f"Pre-set value `{val}` is not within valid values of "
+                        f"parameter `{'/'.join(path)}`: {domain.domain_str}"
+                    )
+            else:
+                # domain is actually a fixed value
+                if domain != val:
+                    logger.warning(
+                        f"Pre-set value `{val}` is not equal to the value of "
+                        f"parameter `{'/'.join(path)}`: {domain}"
+                    )
+        assign_value(spec["config"], path, val)
+
+    return _generate_variants_internal(
+        spec, constant_grid_search=constant_grid_search, random_state=random_state
+    )
+
+
+@DeveloperAPI
+def assign_value(spec: Dict, path: Tuple, value: Any):
+    """Assigns a value to a nested dictionary.
+
+    Handles the special case of tuples, in which case the tuples
+    will be re-constructed to accommodate the updated value.
+    """
+    parent_spec = None
+    parent_key = None
+    for k in path[:-1]:
+        parent_spec = spec
+        parent_key = k
+        spec = spec[k]
+    key = path[-1]
+    if not isinstance(spec, tuple):
+        # spec is mutable. Just assign the value.
+        spec[key] = value
+    else:
+        if parent_spec is None:
+            raise ValueError("Cannot assign value to a tuple.")
+        assert isinstance(key, int), "Tuple key must be an int."
+        # Special handling since tuples are immutable.
+        parent_spec[parent_key] = spec[:key] + (value,) + spec[key + 1 :]
+
+
+def _get_value(spec: Dict, path: Tuple) -> Any:
+    for k in path:
+        spec = spec[k]
+    return spec
+
+
+def _resolve_domain_vars(
+    spec: Dict,
+    domain_vars: List[Tuple[Tuple, Domain]],
+    allow_fail: bool = False,
+    random_state: "RandomState" = None,
+) -> Tuple[bool, Dict]:
+    resolved = {}
+    error = True
+    num_passes = 0
+    while error and num_passes < _MAX_RESOLUTION_PASSES:
+        num_passes += 1
+        error = False
+        for path, domain in domain_vars:
+            if path in resolved:
+                continue
+            try:
+                value = domain.sample(
+                    _UnresolvedAccessGuard(spec), random_state=random_state
+                )
+            except RecursiveDependencyError as e:
+                error = e
+            except Exception:
+                raise ValueError(
+                    "Failed to evaluate expression: {}: {}".format(path, domain)
+                )
+            else:
+                assign_value(spec, path, value)
+                resolved[path] = value
+    if error:
+        if not allow_fail:
+            raise error
+        else:
+            return False, resolved
+    return True, resolved
+
+
+def _grid_search_generator(
+    unresolved_spec: Dict, grid_vars: List
+) -> Generator[Dict, None, None]:
+    value_indices = [0] * len(grid_vars)
+
+    def increment(i):
+        value_indices[i] += 1
+        if value_indices[i] >= len(grid_vars[i][1]):
+            value_indices[i] = 0
+            if i + 1 < len(value_indices):
+                return increment(i + 1)
+            else:
+                return True
+        return False
+
+    if not grid_vars:
+        yield unresolved_spec
+        return
+
+    while value_indices[-1] < len(grid_vars[-1][1]):
+        spec = copy.deepcopy(unresolved_spec)
+        for i, (path, values) in enumerate(grid_vars):
+            assign_value(spec, path, values[value_indices[i]])
+        yield spec
+        if grid_vars:
+            done = increment(0)
+            if done:
+                break
+
+
+def _is_resolved(v) -> bool:
+    resolved, _ = _try_resolve(v)
+    return resolved
+
+
+def _try_resolve(v) -> Tuple[bool, Any]:
+    if isinstance(v, Domain):
+        # Domain to sample from
+        return False, v
+    elif isinstance(v, dict) and len(v) == 1 and "eval" in v:
+        # Lambda function in eval syntax
+        return False, Function(
+            lambda spec: eval(v["eval"], _STANDARD_IMPORTS, {"spec": spec})
+        )
+    elif isinstance(v, dict) and len(v) == 1 and "grid_search" in v:
+        # Grid search values
+        grid_values = v["grid_search"]
+        return False, Categorical(grid_values).grid()
+    return True, v
+
+
+def _split_resolved_unresolved_values(
+    spec: Dict,
+) -> Tuple[Dict[Tuple, Any], Dict[Tuple, Any]]:
+    resolved_vars = {}
+    unresolved_vars = {}
+    for k, v in spec.items():
+        resolved, v = _try_resolve(v)
+        if not resolved:
+            unresolved_vars[(k,)] = v
+        elif isinstance(v, dict):
+            # Recurse into a dict
+            (
+                _resolved_children,
+                _unresolved_children,
+            ) = _split_resolved_unresolved_values(v)
+            for path, value in _resolved_children.items():
+                resolved_vars[(k,) + path] = value
+            for path, value in _unresolved_children.items():
+                unresolved_vars[(k,) + path] = value
+        elif isinstance(v, (list, tuple)):
+            # Recurse into a list
+            for i, elem in enumerate(v):
+                (
+                    _resolved_children,
+                    _unresolved_children,
+                ) = _split_resolved_unresolved_values({i: elem})
+                for path, value in _resolved_children.items():
+                    resolved_vars[(k,) + path] = value
+                for path, value in _unresolved_children.items():
+                    unresolved_vars[(k,) + path] = value
+        else:
+            resolved_vars[(k,)] = v
+    return resolved_vars, unresolved_vars
+
+
+def _unresolved_values(spec: Dict) -> Dict[Tuple, Any]:
+    return _split_resolved_unresolved_values(spec)[1]
+
+
+def _has_unresolved_values(spec: Dict) -> bool:
+    return True if _unresolved_values(spec) else False
+
+
+class _UnresolvedAccessGuard(dict):
+    def __init__(self, *args, **kwds):
+        super(_UnresolvedAccessGuard, self).__init__(*args, **kwds)
+        self.__dict__ = self
+
+    def __getattribute__(self, item):
+        value = dict.__getattribute__(self, item)
+        if not _is_resolved(value):
+            raise RecursiveDependencyError(
+                "`{}` recursively depends on {}".format(item, value)
+            )
+        elif isinstance(value, dict):
+            return _UnresolvedAccessGuard(value)
+        else:
+            return value
+
+
+@DeveloperAPI
+class RecursiveDependencyError(Exception):
+    def __init__(self, msg: str):
+        Exception.__init__(self, msg)

python/ray/tune/search/zoopt/__init__.py

@@ -0,0 +1,3 @@
+from ray.tune.search.zoopt.zoopt_search import ZOOptSearch
+
+__all__ = ["ZOOptSearch"]

python/ray/tune/search/zoopt/zoopt_search.py

@@ -0,0 +1,381 @@
+import copy
+import logging
+from typing import Dict, List, Optional, Tuple
+
+import ray
+import ray.cloudpickle as pickle
+from ray.tune.result import DEFAULT_METRIC
+from ray.tune.search import (
+    UNDEFINED_METRIC_MODE,
+    UNDEFINED_SEARCH_SPACE,
+    UNRESOLVED_SEARCH_SPACE,
+    Searcher,
+)
+from ray.tune.search.sample import (
+    Categorical,
+    Domain,
+    Float,
+    Integer,
+    Quantized,
+    Uniform,
+)
+from ray.tune.search.variant_generator import parse_spec_vars
+from ray.tune.utils.util import unflatten_dict
+
+try:
+    import zoopt
+    from zoopt import Solution, ValueType
+except ImportError:
+    zoopt = None
+    Solution = ValueType = None
+
+logger = logging.getLogger(__name__)
+
+
+class ZOOptSearch(Searcher):
+    """A wrapper around ZOOpt to provide trial suggestions.
+
+    ZOOptSearch is a library for derivative-free optimization. It is backed by
+    the `ZOOpt <https://github.com/polixir/ZOOpt>`__ package. Currently,
+    Asynchronous Sequential RAndomized COordinate Shrinking (ASRacos)
+    is implemented in Tune.
+
+    To use ZOOptSearch, install zoopt (>=0.4.1): ``pip install -U zoopt``.
+
+    Tune automatically converts search spaces to ZOOpt"s format:
+
+    .. code-block:: python
+
+        from ray import tune
+        from ray.tune.search.zoopt import ZOOptSearch
+
+        "config": {
+            "iterations": 10,  # evaluation times
+            "width": tune.uniform(-10, 10),
+            "height": tune.uniform(-10, 10)
+        }
+
+        zoopt_search_config = {
+            "parallel_num": 8,  # how many workers to parallel
+        }
+
+        zoopt_search = ZOOptSearch(
+            algo="Asracos",  # only support Asracos currently
+            budget=20,  # must match `num_samples` in `tune.TuneConfig()`.
+            dim_dict=dim_dict,
+            metric="mean_loss",
+            mode="min",
+            **zoopt_search_config
+        )
+
+        tuner = tune.Tuner(
+            my_objective,
+            tune_config=tune.TuneConfig(
+                search_alg=zoopt_search,
+                num_samples=20
+            ),
+            run_config=tune.RunConfig(
+                name="zoopt_search",
+                stop={"timesteps_total": 10}
+            ),
+            param_space=config
+        )
+        tuner.fit()
+
+    If you would like to pass the search space manually, the code would
+    look like this:
+
+    .. code-block:: python
+
+        from ray import tune
+        from ray.tune.search.zoopt import ZOOptSearch
+        from zoopt import ValueType
+
+        dim_dict = {
+            "height": (ValueType.CONTINUOUS, [-10, 10], 1e-2),
+            "width": (ValueType.DISCRETE, [-10, 10], False),
+            "layers": (ValueType.GRID, [4, 8, 16])
+        }
+
+        "config": {
+            "iterations": 10,  # evaluation times
+        }
+
+        zoopt_search_config = {
+            "parallel_num": 8,  # how many workers to parallel
+        }
+
+        zoopt_search = ZOOptSearch(
+            algo="Asracos",  # only support Asracos currently
+            budget=20,  # must match `num_samples` in `tune.TuneConfig()`.
+            dim_dict=dim_dict,
+            metric="mean_loss",
+            mode="min",
+            **zoopt_search_config
+        )
+
+        tuner = tune.Tuner(
+            my_objective,
+            tune_config=tune.TuneConfig(
+                search_alg=zoopt_search,
+                num_samples=20
+            ),
+            run_config=tune.RunConfig(
+                name="zoopt_search",
+                stop={"timesteps_total": 10}
+            ),
+        )
+        tuner.fit()
+
+    Parameters:
+        algo: To specify an algorithm in zoopt you want to use.
+            Only support ASRacos currently.
+        budget: Number of samples.
+        dim_dict: Dimension dictionary.
+            For continuous dimensions: (continuous, search_range, precision);
+            For discrete dimensions: (discrete, search_range, has_order);
+            For grid dimensions: (grid, grid_list).
+            More details can be found in zoopt package.
+        metric: The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
+        mode: One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute.
+        points_to_evaluate: Initial parameter suggestions to be run
+            first. This is for when you already have some good parameters
+            you want to run first to help the algorithm make better suggestions
+            for future parameters. Needs to be a list of dicts containing the
+            configurations.
+        parallel_num: How many workers to parallel. Note that initial
+            phase may start less workers than this number. More details can
+            be found in zoopt package.
+        **kwargs: Additional keyword arguments forwarded to the underlying
+            zoopt optimizer.
+    """
+
+    optimizer = None
+
+    def __init__(
+        self,
+        algo: str = "asracos",
+        budget: Optional[int] = None,
+        dim_dict: Optional[Dict] = None,
+        metric: Optional[str] = None,
+        mode: Optional[str] = None,
+        points_to_evaluate: Optional[List[Dict]] = None,
+        parallel_num: int = 1,
+        **kwargs
+    ):
+        assert (
+            zoopt is not None
+        ), "ZOOpt not found - please install zoopt by `pip install -U zoopt`."
+        assert budget is not None, "`budget` should not be None!"
+        if mode:
+            assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
+        _algo = algo.lower()
+        assert _algo in [
+            "asracos",
+            "sracos",
+        ], "`algo` must be in ['asracos', 'sracos'] currently"
+
+        self._algo = _algo
+
+        if isinstance(dim_dict, dict) and dim_dict:
+            resolved_vars, domain_vars, grid_vars = parse_spec_vars(dim_dict)
+            if domain_vars or grid_vars:
+                logger.warning(
+                    UNRESOLVED_SEARCH_SPACE.format(par="dim_dict", cls=type(self))
+                )
+                dim_dict = self.convert_search_space(dim_dict, join=True)
+
+        self._dim_dict = dim_dict
+        self._budget = budget
+
+        self._metric = metric
+        if mode == "max":
+            self._metric_op = -1.0
+        elif mode == "min":
+            self._metric_op = 1.0
+
+        self._points_to_evaluate = copy.deepcopy(points_to_evaluate)
+
+        self._live_trial_mapping = {}
+
+        self._dim_keys = []
+        self.solution_dict = {}
+        self.best_solution_list = []
+        self.optimizer = None
+
+        self.kwargs = kwargs
+
+        self.parallel_num = parallel_num
+
+        super(ZOOptSearch, self).__init__(metric=self._metric, mode=mode)
+
+        if self._dim_dict:
+            self._setup_zoopt()
+
+    def _setup_zoopt(self):
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
+        _dim_list = []
+        for k in self._dim_dict:
+            self._dim_keys.append(k)
+            _dim_list.append(self._dim_dict[k])
+
+        init_samples = None
+        if self._points_to_evaluate:
+            logger.warning(
+                "`points_to_evaluate` is ignored by ZOOpt in versions <= 0.4.1."
+            )
+            init_samples = [
+                Solution(x=tuple(point[dim] for dim in self._dim_keys))
+                for point in self._points_to_evaluate
+            ]
+        dim = zoopt.Dimension2(_dim_list)
+        par = zoopt.Parameter(budget=self._budget, init_samples=init_samples)
+        if self._algo == "sracos" or self._algo == "asracos":
+            from zoopt.algos.opt_algorithms.racos.sracos import SRacosTune
+
+            self.optimizer = SRacosTune(
+                dimension=dim,
+                parameter=par,
+                parallel_num=self.parallel_num,
+                **self.kwargs
+            )
+
+    def set_search_properties(
+        self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
+    ) -> bool:
+        if self._dim_dict:
+            return False
+        space = self.convert_search_space(config)
+        self._dim_dict = space
+
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        if self._mode == "max":
+            self._metric_op = -1.0
+        elif self._mode == "min":
+            self._metric_op = 1.0
+
+        self._setup_zoopt()
+        return True
+
+    def suggest(self, trial_id: str) -> Optional[Dict]:
+        if not self._dim_dict or not self.optimizer:
+            raise RuntimeError(
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="dim_dict"
+                )
+            )
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__, metric=self._metric, mode=self._mode
+                )
+            )
+
+        _solution = self.optimizer.suggest()
+
+        if _solution == "FINISHED":
+            if ray.__version__ >= "0.8.7":
+                return Searcher.FINISHED
+            else:
+                return None
+
+        if _solution:
+            self.solution_dict[str(trial_id)] = _solution
+            _x = _solution.get_x()
+            new_trial = dict(zip(self._dim_keys, _x))
+            self._live_trial_mapping[trial_id] = new_trial
+            return unflatten_dict(new_trial)
+
+    def on_trial_complete(
+        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
+    ):
+        """Notification for the completion of trial."""
+        if result:
+            _solution = self.solution_dict[str(trial_id)]
+            _best_solution_so_far = self.optimizer.complete(
+                _solution, self._metric_op * result[self._metric]
+            )
+            if _best_solution_so_far:
+                self.best_solution_list.append(_best_solution_so_far)
+
+        del self._live_trial_mapping[trial_id]
+
+    def save(self, checkpoint_path: str):
+        save_object = self.__dict__
+        with open(checkpoint_path, "wb") as outputFile:
+            pickle.dump(save_object, outputFile)
+
+    def restore(self, checkpoint_path: str):
+        with open(checkpoint_path, "rb") as inputFile:
+            save_object = pickle.load(inputFile)
+        self.__dict__.update(save_object)
+
+    @staticmethod
+    def convert_search_space(spec: Dict, join: bool = False) -> Dict[str, Tuple]:
+        spec = copy.deepcopy(spec)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if not domain_vars and not grid_vars:
+            return {}
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a ZOOpt search space."
+            )
+
+        def resolve_value(domain: Domain) -> Tuple:
+            quantize = None
+
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                quantize = sampler.q
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                precision = quantize or 1e-12
+                if isinstance(sampler, Uniform):
+                    return (
+                        ValueType.CONTINUOUS,
+                        [domain.lower, domain.upper],
+                        precision,
+                    )
+
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, Uniform):
+                    return (ValueType.DISCRETE, [domain.lower, domain.upper - 1], True)
+
+            elif isinstance(domain, Categorical):
+                # Categorical variables would use ValueType.DISCRETE with
+                # has_partial_order=False, however, currently we do not
+                # keep track of category values and cannot automatically
+                # translate back and forth between them.
+                if isinstance(sampler, Uniform):
+                    return (ValueType.GRID, domain.categories)
+
+            raise ValueError(
+                "ZOOpt does not support parameters of type "
+                "`{}` with samplers of type `{}`".format(
+                    type(domain).__name__, type(domain.sampler).__name__
+                )
+            )
+
+        conv_spec = {
+            "/".join(path): resolve_value(domain) for path, domain in domain_vars
+        }
+
+        if join:
+            spec.update(conv_spec)
+            conv_spec = spec
+
+        return conv_spec