wehub-resource-sync
863 lines
35 KiB
Python
863 lines
35 KiB
Python
import collections
|
|
import os
|
|
import unittest
|
|
from unittest.mock import MagicMock, Mock, patch
|
|
|
|
import numpy as np
|
|
import pytest
|
|
import regex as re
|
|
|
|
from ray import tune
|
|
from ray._common.test_utils import run_string_as_driver
|
|
from ray.tune.experiment.trial import Trial
|
|
from ray.tune.progress_reporter import (
|
|
CLIReporter,
|
|
JupyterNotebookReporter,
|
|
ProgressReporter,
|
|
TuneReporterBase,
|
|
_best_trial_str,
|
|
_detect_reporter,
|
|
_fair_filter_trials,
|
|
_max_len,
|
|
_time_passed_str,
|
|
_trial_progress_str,
|
|
)
|
|
from ray.tune.result import AUTO_RESULT_KEYS
|
|
|
|
EXPECTED_RESULT_1 = """Result logdir: /foo
|
|
Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED)
|
|
+--------------+------------+-------+-----+-----+------------+
|
|
| Trial name | status | loc | a | b | metric_1 |
|
|
|--------------+------------+-------+-----+-----+------------|
|
|
| 00002 | RUNNING | here | 2 | 4 | 1 |
|
|
| 00001 | PENDING | here | 1 | 2 | 0.5 |
|
|
| 00000 | TERMINATED | here | 0 | 0 | 0 |
|
|
+--------------+------------+-------+-----+-----+------------+
|
|
... 2 more trials not shown (2 RUNNING)"""
|
|
|
|
EXPECTED_RESULT_2 = """Result logdir: /foo
|
|
Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED)
|
|
+--------------+------------+-------+-----+-----+---------+---------+
|
|
| Trial name | status | loc | a | b | n/k/0 | n/k/1 |
|
|
|--------------+------------+-------+-----+-----+---------+---------|
|
|
| 00002 | RUNNING | here | 2 | 4 | 2 | 4 |
|
|
| 00003 | RUNNING | here | 3 | 6 | 3 | 6 |
|
|
| 00004 | RUNNING | here | 4 | 8 | 4 | 8 |
|
|
| 00001 | PENDING | here | 1 | 2 | 1 | 2 |
|
|
| 00000 | TERMINATED | here | 0 | 0 | 0 | 0 |
|
|
+--------------+------------+-------+-----+-----+---------+---------+"""
|
|
|
|
EXPECTED_RESULT_3 = """Result logdir: /foo
|
|
Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED)
|
|
+--------------+------------+-------+-----+-----------+------------+
|
|
| Trial name | status | loc | A | NestSub | Metric 2 |
|
|
|--------------+------------+-------+-----+-----------+------------|
|
|
| 00002 | RUNNING | here | 2 | 1 | 0.5 |
|
|
| 00001 | PENDING | here | 1 | 0.5 | 0.25 |
|
|
| 00000 | TERMINATED | here | 0 | 0 | 0 |
|
|
+--------------+------------+-------+-----+-----------+------------+
|
|
... 2 more trials not shown (2 RUNNING)"""
|
|
|
|
EXPECTED_RESULT_4 = """Result logdir: /foo
|
|
Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED)
|
|
+--------------+------------+-------+-----+-----+------------+
|
|
| Trial name | status | loc | a | b | metric_1 |
|
|
|--------------+------------+-------+-----+-----+------------|
|
|
| 00002 | RUNNING | here | 2 | 4 | 1 |
|
|
| 00003 | RUNNING | here | 3 | 6 | 1.5 |
|
|
| 00004 | RUNNING | here | 4 | 8 | 2 |
|
|
| 00001 | PENDING | here | 1 | 2 | 0.5 |
|
|
| 00000 | TERMINATED | here | 0 | 0 | 0 |
|
|
+--------------+------------+-------+-----+-----+------------+"""
|
|
|
|
END_TO_END_COMMAND = """
|
|
import ray
|
|
from ray import tune
|
|
from ray.tune.experiment.trial import _Location
|
|
from ray.tune.progress_reporter import _get_trial_location
|
|
from unittest.mock import patch
|
|
|
|
|
|
def mock_get_trial_location(trial, result):
|
|
location = _get_trial_location(trial, result)
|
|
if location.pid:
|
|
return _Location("123.123.123.123", "1")
|
|
return location
|
|
|
|
|
|
with patch("ray.tune.progress_reporter._get_trial_location",
|
|
mock_get_trial_location):
|
|
reporter = tune.progress_reporter.CLIReporter(metric_columns=["done"])
|
|
|
|
def f(config):
|
|
return {"done": True}
|
|
|
|
ray.init(num_cpus=1)
|
|
tune.run_experiments(
|
|
{
|
|
"one": {
|
|
"run": f,
|
|
"config": {
|
|
"a": tune.grid_search(list(range(10))),
|
|
},
|
|
},
|
|
"two": {
|
|
"run": f,
|
|
"config": {
|
|
"b": tune.grid_search(list(range(10))),
|
|
},
|
|
},
|
|
"three": {
|
|
"run": f,
|
|
"config": {
|
|
"c": tune.grid_search(list(range(10))),
|
|
},
|
|
},
|
|
},
|
|
verbose=3,
|
|
progress_reporter=reporter)"""
|
|
|
|
EXPECTED_END_TO_END_START = """Number of trials: 30/30 (29 PENDING, 1 RUNNING)
|
|
+---------------+----------+-------------------+-----+-----+
|
|
| Trial name | status | loc | a | b |
|
|
|---------------+----------+-------------------+-----+-----|
|
|
| f_xxxxx_00000 | RUNNING | 123.123.123.123:1 | 0 | |
|
|
| f_xxxxx_00001 | PENDING | | 1 | |"""
|
|
|
|
EXPECTED_END_TO_END_END = """Number of trials: 30/30 (30 TERMINATED)
|
|
+---------------+------------+-------------------+-----+-----+-----+--------+
|
|
| Trial name | status | loc | a | b | c | done |
|
|
|---------------+------------+-------------------+-----+-----+-----+--------|
|
|
| f_xxxxx_00000 | TERMINATED | 123.123.123.123:1 | 0 | | | True |
|
|
| f_xxxxx_00001 | TERMINATED | 123.123.123.123:1 | 1 | | | True |
|
|
| f_xxxxx_00002 | TERMINATED | 123.123.123.123:1 | 2 | | | True |
|
|
| f_xxxxx_00003 | TERMINATED | 123.123.123.123:1 | 3 | | | True |
|
|
| f_xxxxx_00004 | TERMINATED | 123.123.123.123:1 | 4 | | | True |
|
|
| f_xxxxx_00005 | TERMINATED | 123.123.123.123:1 | 5 | | | True |
|
|
| f_xxxxx_00006 | TERMINATED | 123.123.123.123:1 | 6 | | | True |
|
|
| f_xxxxx_00007 | TERMINATED | 123.123.123.123:1 | 7 | | | True |
|
|
| f_xxxxx_00008 | TERMINATED | 123.123.123.123:1 | 8 | | | True |
|
|
| f_xxxxx_00009 | TERMINATED | 123.123.123.123:1 | 9 | | | True |
|
|
| f_xxxxx_00010 | TERMINATED | 123.123.123.123:1 | | 0 | | True |
|
|
| f_xxxxx_00011 | TERMINATED | 123.123.123.123:1 | | 1 | | True |
|
|
| f_xxxxx_00012 | TERMINATED | 123.123.123.123:1 | | 2 | | True |
|
|
| f_xxxxx_00013 | TERMINATED | 123.123.123.123:1 | | 3 | | True |
|
|
| f_xxxxx_00014 | TERMINATED | 123.123.123.123:1 | | 4 | | True |
|
|
| f_xxxxx_00015 | TERMINATED | 123.123.123.123:1 | | 5 | | True |
|
|
| f_xxxxx_00016 | TERMINATED | 123.123.123.123:1 | | 6 | | True |
|
|
| f_xxxxx_00017 | TERMINATED | 123.123.123.123:1 | | 7 | | True |
|
|
| f_xxxxx_00018 | TERMINATED | 123.123.123.123:1 | | 8 | | True |
|
|
| f_xxxxx_00019 | TERMINATED | 123.123.123.123:1 | | 9 | | True |
|
|
| f_xxxxx_00020 | TERMINATED | 123.123.123.123:1 | | | 0 | True |
|
|
| f_xxxxx_00021 | TERMINATED | 123.123.123.123:1 | | | 1 | True |
|
|
| f_xxxxx_00022 | TERMINATED | 123.123.123.123:1 | | | 2 | True |
|
|
| f_xxxxx_00023 | TERMINATED | 123.123.123.123:1 | | | 3 | True |
|
|
| f_xxxxx_00024 | TERMINATED | 123.123.123.123:1 | | | 4 | True |
|
|
| f_xxxxx_00025 | TERMINATED | 123.123.123.123:1 | | | 5 | True |
|
|
| f_xxxxx_00026 | TERMINATED | 123.123.123.123:1 | | | 6 | True |
|
|
| f_xxxxx_00027 | TERMINATED | 123.123.123.123:1 | | | 7 | True |
|
|
| f_xxxxx_00028 | TERMINATED | 123.123.123.123:1 | | | 8 | True |
|
|
| f_xxxxx_00029 | TERMINATED | 123.123.123.123:1 | | | 9 | True |
|
|
+---------------+------------+-------------------+-----+-----+-----+--------+""" # noqa
|
|
|
|
EXPECTED_END_TO_END_AC = """Number of trials: 30/30 (30 TERMINATED)
|
|
+---------------+------------+-------+-----+-----+-----+
|
|
| Trial name | status | loc | a | b | c |
|
|
|---------------+------------+-------+-----+-----+-----|
|
|
| f_xxxxx_00000 | TERMINATED | | 0 | | |
|
|
| f_xxxxx_00001 | TERMINATED | | 1 | | |
|
|
| f_xxxxx_00002 | TERMINATED | | 2 | | |
|
|
| f_xxxxx_00003 | TERMINATED | | 3 | | |
|
|
| f_xxxxx_00004 | TERMINATED | | 4 | | |
|
|
| f_xxxxx_00005 | TERMINATED | | 5 | | |
|
|
| f_xxxxx_00006 | TERMINATED | | 6 | | |
|
|
| f_xxxxx_00007 | TERMINATED | | 7 | | |
|
|
| f_xxxxx_00008 | TERMINATED | | 8 | | |
|
|
| f_xxxxx_00009 | TERMINATED | | 9 | | |
|
|
| f_xxxxx_00010 | TERMINATED | | | 0 | |
|
|
| f_xxxxx_00011 | TERMINATED | | | 1 | |
|
|
| f_xxxxx_00012 | TERMINATED | | | 2 | |
|
|
| f_xxxxx_00013 | TERMINATED | | | 3 | |
|
|
| f_xxxxx_00014 | TERMINATED | | | 4 | |
|
|
| f_xxxxx_00015 | TERMINATED | | | 5 | |
|
|
| f_xxxxx_00016 | TERMINATED | | | 6 | |
|
|
| f_xxxxx_00017 | TERMINATED | | | 7 | |
|
|
| f_xxxxx_00018 | TERMINATED | | | 8 | |
|
|
| f_xxxxx_00019 | TERMINATED | | | 9 | |
|
|
| f_xxxxx_00020 | TERMINATED | | | | 0 |
|
|
| f_xxxxx_00021 | TERMINATED | | | | 1 |
|
|
| f_xxxxx_00022 | TERMINATED | | | | 2 |
|
|
| f_xxxxx_00023 | TERMINATED | | | | 3 |
|
|
| f_xxxxx_00024 | TERMINATED | | | | 4 |
|
|
| f_xxxxx_00025 | TERMINATED | | | | 5 |
|
|
| f_xxxxx_00026 | TERMINATED | | | | 6 |
|
|
| f_xxxxx_00027 | TERMINATED | | | | 7 |
|
|
| f_xxxxx_00028 | TERMINATED | | | | 8 |
|
|
| f_xxxxx_00029 | TERMINATED | | | | 9 |
|
|
+---------------+------------+-------+-----+-----+-----+"""
|
|
|
|
EXPECTED_BEST_1 = (
|
|
"Current best trial: 00001 with metric_1=0.5 and "
|
|
"parameters={'a': 1, 'b': 2, 'n': {'k': [1, 2]}}"
|
|
)
|
|
|
|
EXPECTED_BEST_2 = "Current best trial: 00004 with metric_1=2.0 and parameters={'a': 4}"
|
|
|
|
EXPECTED_SORT_RESULT_UNSORTED = """Number of trials: 5 (1 PENDING, 1 RUNNING, 3 TERMINATED)
|
|
+--------------+------------+-------+-----+------------+
|
|
| Trial name | status | loc | a | metric_1 |
|
|
|--------------+------------+-------+-----+------------|
|
|
| 00004 | RUNNING | here | 4 | |
|
|
| 00003 | PENDING | here | 3 | |
|
|
| 00000 | TERMINATED | here | 0 | 0.3 |
|
|
| 00001 | TERMINATED | here | 1 | 0.2 |
|
|
+--------------+------------+-------+-----+------------+
|
|
... 1 more trials not shown (1 TERMINATED)"""
|
|
|
|
EXPECTED_SORT_RESULT_ASC = """Number of trials: 5 (1 PENDING, 1 RUNNING, 3 TERMINATED)
|
|
+--------------+------------+-------+-----+------------+
|
|
| Trial name | status | loc | a | metric_1 |
|
|
|--------------+------------+-------+-----+------------|
|
|
| 00004 | RUNNING | here | 4 | |
|
|
| 00003 | PENDING | here | 3 | |
|
|
| 00001 | TERMINATED | here | 1 | 0.2 |
|
|
| 00000 | TERMINATED | here | 0 | 0.3 |
|
|
+--------------+------------+-------+-----+------------+
|
|
... 1 more trials not shown (1 TERMINATED)"""
|
|
|
|
EXPECTED_NESTED_SORT_RESULT = """Number of trials: 5 (1 PENDING, 1 RUNNING, 3 TERMINATED)
|
|
+--------------+------------+-------+-----+-------------------+
|
|
| Trial name | status | loc | a | nested/metric_2 |
|
|
|--------------+------------+-------+-----+-------------------|
|
|
| 00004 | RUNNING | here | 4 | |
|
|
| 00003 | PENDING | here | 3 | |
|
|
| 00001 | TERMINATED | here | 1 | 0.2 |
|
|
| 00000 | TERMINATED | here | 0 | 0.3 |
|
|
+--------------+------------+-------+-----+-------------------+
|
|
... 1 more trials not shown (1 TERMINATED)"""
|
|
|
|
EXPECTED_SORT_RESULT_DESC = """Number of trials: 5 (1 PENDING, 1 RUNNING, 3 TERMINATED)
|
|
+--------------+------------+-------+-----+------------+
|
|
| Trial name | status | loc | a | metric_1 |
|
|
|--------------+------------+-------+-----+------------|
|
|
| 00004 | RUNNING | here | 4 | |
|
|
| 00003 | PENDING | here | 3 | |
|
|
| 00002 | TERMINATED | here | 2 | 0.4 |
|
|
| 00000 | TERMINATED | here | 0 | 0.3 |
|
|
+--------------+------------+-------+-----+------------+
|
|
... 1 more trials not shown (1 TERMINATED)"""
|
|
|
|
VERBOSE_EXP_OUT_1 = "Number of trials: 3/3 (2 PENDING, 1 RUNNING)"
|
|
VERBOSE_EXP_OUT_2 = "Number of trials: 3/3 (3 TERMINATED)"
|
|
|
|
VERBOSE_TRIAL_NORM_1 = (
|
|
"Trial train_fn_xxxxx_00000 reported acc=5 "
|
|
"with parameters={'do': 'complete'}. This trial completed.\n"
|
|
)
|
|
|
|
# NOTE: We use Regex for `VERBOSE_TRIAL_NORM_2` to make the test deterministic.
|
|
# `"Trial train_fn_xxxxx_00001 reported..."` and
|
|
# `"Trial train_fn_xxxxx_00001 completed..."`
|
|
# are printed in separate calls. Sometimes, a status update is printed between the
|
|
# calls. For more information, see #29693.
|
|
VERBOSE_TRIAL_NORM_2_PATTERN = (
|
|
r"Trial train_fn_xxxxx_00001 reported _metric=6 "
|
|
r"with parameters=\{'do': 'once'\}\.\n"
|
|
r"(?s).*"
|
|
r"Trial train_fn_xxxxx_00001 completed\. Last result: _metric=6\n"
|
|
)
|
|
|
|
VERBOSE_TRIAL_NORM_3 = (
|
|
"Trial train_fn_xxxxx_00002 reported acc=7 with parameters={'do': 'twice'}.\n"
|
|
)
|
|
|
|
VERBOSE_TRIAL_NORM_4 = (
|
|
"Trial train_fn_xxxxx_00002 reported acc=8 "
|
|
"with parameters={'do': 'twice'}. This trial completed.\n"
|
|
)
|
|
|
|
VERBOSE_TRIAL_WITH_ONCE_RESULT = "Result for train_fn_xxxxx_00001"
|
|
VERBOSE_TRIAL_WITH_ONCE_COMPLETED = "Trial train_fn_xxxxx_00001 completed."
|
|
|
|
VERBOSE_TRIAL_DETAIL = """+-------------------+----------+-------------------+----------+
|
|
| Trial name | status | loc | do |
|
|
|-------------------+----------+-------------------+----------|
|
|
| train_fn_xxxxx_00000 | RUNNING | 123.123.123.123:1 | complete |"""
|
|
|
|
VERBOSE_CMD = """import ray.tune
|
|
import random
|
|
import numpy as np
|
|
import time
|
|
from ray.tune.experiment.trial import _Location
|
|
from ray.tune.progress_reporter import _get_trial_location
|
|
from unittest.mock import patch
|
|
|
|
|
|
def mock_get_trial_location(trial, result):
|
|
location = _get_trial_location(trial, result)
|
|
if location.pid:
|
|
return _Location("123.123.123.123", "1")
|
|
return location
|
|
|
|
|
|
def train_fn(config):
|
|
if config["do"] == "complete":
|
|
time.sleep(0.1)
|
|
ray.tune.report(dict(acc=5, done=True))
|
|
elif config["do"] == "once":
|
|
time.sleep(0.5)
|
|
return 6
|
|
else:
|
|
time.sleep(1.0)
|
|
ray.tune.report(dict(acc=7))
|
|
ray.tune.report(dict(acc=8))
|
|
|
|
random.seed(1234)
|
|
np.random.seed(1234)
|
|
|
|
|
|
with patch("ray.tune.progress_reporter._get_trial_location",
|
|
mock_get_trial_location):
|
|
ray.tune.run(
|
|
train_fn,
|
|
config={
|
|
"do": ray.tune.grid_search(["complete", "once", "twice"])
|
|
},"""
|
|
|
|
# Add "verbose=3)" etc
|
|
|
|
|
|
class ProgressReporterTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
os.environ["TUNE_MAX_PENDING_TRIALS_PG"] = "auto"
|
|
os.environ["RAY_AIR_NEW_OUTPUT"] = "0"
|
|
|
|
def mock_trial(self, status, i):
|
|
mock = MagicMock()
|
|
mock.status = status
|
|
mock.trial_id = "%05d" % i
|
|
return mock
|
|
|
|
def testFairFilterTrials(self):
|
|
"""Tests that trials are represented fairly."""
|
|
trials_by_state = collections.defaultdict(list)
|
|
# States for which trials are under and overrepresented
|
|
states_under = (Trial.PAUSED, Trial.ERROR)
|
|
states_over = (Trial.PENDING, Trial.RUNNING, Trial.TERMINATED)
|
|
|
|
max_trials = 13
|
|
num_trials_under = 2 # num of trials for each underrepresented state
|
|
num_trials_over = 10 # num of trials for each overrepresented state
|
|
|
|
i = 0
|
|
for state in states_under:
|
|
for _ in range(num_trials_under):
|
|
trials_by_state[state].append(self.mock_trial(state, i))
|
|
i += 1
|
|
for state in states_over:
|
|
for _ in range(num_trials_over):
|
|
trials_by_state[state].append(self.mock_trial(state, i))
|
|
i += 1
|
|
|
|
filtered_trials_by_state = _fair_filter_trials(
|
|
trials_by_state, max_trials=max_trials
|
|
)
|
|
for state in trials_by_state:
|
|
if state in states_under:
|
|
expected_num_trials = num_trials_under
|
|
else:
|
|
expected_num_trials = (
|
|
max_trials - num_trials_under * len(states_under)
|
|
) / len(states_over)
|
|
state_trials = filtered_trials_by_state[state]
|
|
self.assertEqual(len(state_trials), expected_num_trials)
|
|
# Make sure trials are sorted newest-first within state.
|
|
for i in range(len(state_trials) - 1):
|
|
assert state_trials[i].trial_id < state_trials[i + 1].trial_id
|
|
|
|
def testAddMetricColumn(self):
|
|
"""Tests edge cases of add_metric_column."""
|
|
|
|
# Test list-initialized metric columns.
|
|
reporter = CLIReporter(metric_columns=["foo", "bar"])
|
|
with self.assertRaises(ValueError):
|
|
reporter.add_metric_column("bar")
|
|
|
|
with self.assertRaises(ValueError):
|
|
reporter.add_metric_column("baz", "qux")
|
|
|
|
reporter.add_metric_column("baz")
|
|
self.assertIn("baz", reporter._metric_columns)
|
|
|
|
# Test default-initialized (dict) metric columns.
|
|
reporter = CLIReporter()
|
|
reporter.add_metric_column("foo", "bar")
|
|
self.assertIn("foo", reporter._metric_columns)
|
|
|
|
def testInfer(self):
|
|
reporter = CLIReporter()
|
|
test_result = dict(foo_result=1, baz_result=4123, bar_result="testme")
|
|
|
|
def test(config):
|
|
for i in range(3):
|
|
tune.report(test_result)
|
|
|
|
analysis = tune.run(test, num_samples=3, verbose=3)
|
|
all_trials = analysis.trials
|
|
inferred_results = reporter._infer_user_metrics(all_trials)
|
|
|
|
for metric in inferred_results:
|
|
self.assertNotIn(metric, AUTO_RESULT_KEYS)
|
|
self.assertTrue(metric in test_result)
|
|
|
|
class TestReporter(CLIReporter):
|
|
_output = []
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self._max_report_freqency = 0
|
|
|
|
def report(self, *args, **kwargs):
|
|
progress_str = self._progress_str(*args, **kwargs)
|
|
self._output.append(progress_str)
|
|
|
|
reporter = TestReporter()
|
|
analysis = tune.run(test, num_samples=3, progress_reporter=reporter, verbose=3)
|
|
found = {k: False for k in test_result}
|
|
for output in reporter._output:
|
|
for key in test_result:
|
|
if key in output:
|
|
found[key] = True
|
|
assert found["foo_result"]
|
|
assert found["baz_result"]
|
|
assert not found["bar_result"]
|
|
|
|
def testProgressStr(self):
|
|
trials = []
|
|
for i in range(5):
|
|
t = Mock()
|
|
if i == 0:
|
|
t.status = "TERMINATED"
|
|
elif i == 1:
|
|
t.status = "PENDING"
|
|
else:
|
|
t.status = "RUNNING"
|
|
t.trial_id = "%05d" % i
|
|
t.local_experiment_path = "/foo"
|
|
t.temporary_state = Mock()
|
|
t.temporary_state.location = "here"
|
|
t.config = {"a": i, "b": i * 2, "n": {"k": [i, 2 * i]}}
|
|
t.evaluated_params = {"a": i, "b": i * 2, "n/k/0": i, "n/k/1": 2 * i}
|
|
t.last_result = {
|
|
"config": {"a": i, "b": i * 2, "n": {"k": [i, 2 * i]}},
|
|
"metric_1": i / 2,
|
|
"metric_2": i / 4,
|
|
"nested": {"sub": i / 2},
|
|
}
|
|
t.__str__ = lambda self: self.trial_id
|
|
trials.append(t)
|
|
# One metric, two parameters
|
|
prog1 = _trial_progress_str(
|
|
trials, ["metric_1"], ["a", "b"], fmt="psql", max_rows=3, force_table=True
|
|
)
|
|
print(prog1)
|
|
assert prog1 == EXPECTED_RESULT_1
|
|
|
|
# No metric, all parameters
|
|
prog2 = _trial_progress_str(
|
|
trials, [], None, fmt="psql", max_rows=None, force_table=True
|
|
)
|
|
print(prog2)
|
|
assert prog2 == EXPECTED_RESULT_2
|
|
|
|
# Two metrics, one parameter, all with custom representation
|
|
prog3 = _trial_progress_str(
|
|
trials,
|
|
{"nested/sub": "NestSub", "metric_2": "Metric 2"},
|
|
{"a": "A"},
|
|
fmt="psql",
|
|
max_rows=3,
|
|
force_table=True,
|
|
)
|
|
print(prog3)
|
|
assert prog3 == EXPECTED_RESULT_3
|
|
|
|
# Current best trial
|
|
best1 = _best_trial_str(trials[1], "metric_1")
|
|
assert best1 == EXPECTED_BEST_1
|
|
|
|
def testBestTrialStr(self):
|
|
"""Assert that custom nested parameter columns are printed correctly"""
|
|
config = {"nested": {"conf": "nested_value"}, "toplevel": "toplevel_value"}
|
|
|
|
trial = Trial("", config=config, stub=True)
|
|
trial.run_metadata.last_result = {
|
|
"metric": 1,
|
|
"config": config,
|
|
"nested": {"metric": 2},
|
|
}
|
|
|
|
result = _best_trial_str(trial, "metric")
|
|
self.assertIn("nested_value", result)
|
|
|
|
result = _best_trial_str(trial, "metric", parameter_columns=["nested/conf"])
|
|
self.assertIn("nested_value", result)
|
|
|
|
# Test that this works with a nested metric
|
|
result = _best_trial_str(
|
|
trial, "nested/metric", parameter_columns=["nested/conf"]
|
|
)
|
|
self.assertIn("nested_value", result)
|
|
|
|
def testBestTrialZero(self):
|
|
trial1 = Trial("", config={}, stub=True)
|
|
trial1.run_metadata.last_result = {"metric": 7, "config": {}}
|
|
|
|
trial2 = Trial("", config={}, stub=True)
|
|
trial2.run_metadata.last_result = {"metric": 0, "config": {}}
|
|
|
|
trial3 = Trial("", config={}, stub=True)
|
|
trial3.run_metadata.last_result = {"metric": 2, "config": {}}
|
|
|
|
reporter = TuneReporterBase(metric="metric", mode="min")
|
|
best_trial, metric = reporter._current_best_trial([trial1, trial2, trial3])
|
|
assert best_trial == trial2
|
|
|
|
def testBestTrialNan(self):
|
|
trial1 = Trial("", config={}, stub=True)
|
|
trial1.run_metadata.last_result = {"metric": np.nan, "config": {}}
|
|
|
|
trial2 = Trial("", config={}, stub=True)
|
|
trial2.run_metadata.last_result = {"metric": 0, "config": {}}
|
|
|
|
trial3 = Trial("", config={}, stub=True)
|
|
trial3.run_metadata.last_result = {"metric": 2, "config": {}}
|
|
|
|
reporter = TuneReporterBase(metric="metric", mode="min")
|
|
best_trial, metric = reporter._current_best_trial([trial1, trial2, trial3])
|
|
assert best_trial == trial2
|
|
|
|
trial1 = Trial("", config={}, stub=True)
|
|
trial1.run_metadata.last_result = {"metric": np.nan, "config": {}}
|
|
|
|
trial2 = Trial("", config={}, stub=True)
|
|
trial2.run_metadata.last_result = {"metric": 0, "config": {}}
|
|
|
|
trial3 = Trial("", config={}, stub=True)
|
|
trial3.run_metadata.last_result = {"metric": 2, "config": {}}
|
|
|
|
reporter = TuneReporterBase(metric="metric", mode="max")
|
|
best_trial, metric = reporter._current_best_trial([trial1, trial2, trial3])
|
|
assert best_trial == trial3
|
|
|
|
def testTimeElapsed(self):
|
|
# Sun Feb 7 14:18:40 2016 -0800
|
|
# (time of the first Ray commit)
|
|
time_start = 1454825920
|
|
time_now = (
|
|
time_start
|
|
+ 1 * 60 * 60 # 1 hour
|
|
+ 31 * 60 # 31 minutes
|
|
+ 22 # 22 seconds
|
|
) # time to second commit
|
|
|
|
# Local timezone output can be tricky, so we don't check the
|
|
# day and the hour in this test.
|
|
output = _time_passed_str(time_start, time_now)
|
|
self.assertIn("Current time: 2016-02-", output)
|
|
self.assertIn(":50:02 (running for 01:31:22.00)", output)
|
|
|
|
time_now += 2 * 60 * 60 * 24 # plus two days
|
|
output = _time_passed_str(time_start, time_now)
|
|
self.assertIn("Current time: 2016-02-", output)
|
|
self.assertIn(":50:02 (running for 2 days, 01:31:22.00)", output)
|
|
|
|
def testCurrentBestTrial(self):
|
|
trials = []
|
|
for i in range(5):
|
|
t = Mock()
|
|
t.status = "RUNNING"
|
|
t.trial_id = "%05d" % i
|
|
t.local_experiment_path = "/foo"
|
|
t.temporary_state = Mock()
|
|
t.temporary_state.location = "here"
|
|
t.config = {"a": i, "b": i * 2, "n": {"k": [i, 2 * i]}}
|
|
t.evaluated_params = {"a": i}
|
|
t.last_result = {"config": {"a": i}, "metric_1": i / 2}
|
|
t.__str__ = lambda self: self.trial_id
|
|
trials.append(t)
|
|
|
|
class TestReporter(CLIReporter):
|
|
_output = []
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self._max_report_freqency = 0
|
|
|
|
def report(self, *args, **kwargs):
|
|
progress_str = self._progress_str(*args, **kwargs)
|
|
self._output.append(progress_str)
|
|
|
|
reporter = TestReporter(mode="max")
|
|
reporter.report(trials, done=False)
|
|
assert EXPECTED_BEST_2 in reporter._output[0]
|
|
|
|
def testSortByMetric(self):
|
|
trials = []
|
|
for i in range(5):
|
|
t = Mock()
|
|
if i < 3:
|
|
t.status = "TERMINATED"
|
|
elif i == 3:
|
|
t.status = "PENDING"
|
|
else:
|
|
t.status = "RUNNING"
|
|
t.trial_id = "%05d" % i
|
|
t.local_experiment_path = "/foo"
|
|
t.temporary_state = Mock()
|
|
t.temporary_state.location = "here"
|
|
t.run_metadata = Mock()
|
|
t.config = {"a": i}
|
|
t.evaluated_params = {"a": i}
|
|
t.last_result = {"config": {"a": i}}
|
|
t.__str__ = lambda self: self.trial_id
|
|
trials.append(t)
|
|
# Set `metric_1` for terminated trails
|
|
trials[0].last_result["metric_1"] = 0.3
|
|
trials[0].last_result["nested"] = {"metric_2": 0.3}
|
|
trials[1].last_result["metric_1"] = 0.2
|
|
trials[1].last_result["nested"] = {"metric_2": 0.2}
|
|
trials[2].last_result["metric_1"] = 0.4
|
|
trials[2].last_result["nested"] = {"metric_2": 0.4}
|
|
|
|
class TestReporter(CLIReporter):
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self._max_report_freqency = 0
|
|
self._output = ""
|
|
|
|
def report(self, *args, **kwargs):
|
|
progress_str = self._progress_str(*args, **kwargs)
|
|
self._output = progress_str
|
|
|
|
# Default reporter
|
|
reporter1 = TestReporter(max_progress_rows=4, mode="max", metric="metric_1")
|
|
reporter1.report(trials, done=False)
|
|
assert EXPECTED_SORT_RESULT_UNSORTED in reporter1._output
|
|
|
|
# Sort by metric (asc)
|
|
reporter2 = TestReporter(
|
|
max_progress_rows=4, mode="min", metric="metric_1", sort_by_metric=True
|
|
)
|
|
reporter2.report(trials, done=False)
|
|
assert EXPECTED_SORT_RESULT_ASC in reporter2._output
|
|
|
|
# Sort by metric (desc)
|
|
reporter3 = TestReporter(
|
|
max_progress_rows=4, mode="max", metric="metric_1", sort_by_metric=True
|
|
)
|
|
reporter3.report(trials, done=False)
|
|
assert EXPECTED_SORT_RESULT_DESC in reporter3._output
|
|
|
|
# Sort by metric when mode is None
|
|
reporter4 = TestReporter(
|
|
max_progress_rows=4, metric="metric_1", sort_by_metric=True
|
|
)
|
|
reporter4.report(trials, done=False)
|
|
assert EXPECTED_SORT_RESULT_UNSORTED in reporter4._output
|
|
|
|
# Sort by metric when metric is None
|
|
reporter5 = TestReporter(max_progress_rows=4, mode="max", sort_by_metric=True)
|
|
reporter5.report(trials, done=False)
|
|
assert EXPECTED_SORT_RESULT_UNSORTED in reporter5._output
|
|
|
|
# Sort by metric when metric is passed using
|
|
# reporter.setup (called from tune.run)
|
|
# calling repoter.set_search_properties
|
|
reporter6 = TestReporter(max_progress_rows=4, sort_by_metric=True)
|
|
reporter6.set_search_properties(metric="metric_1", mode="max")
|
|
reporter6.report(trials, done=False)
|
|
assert EXPECTED_SORT_RESULT_DESC in reporter6._output
|
|
|
|
# Sort by nested metric (asc)
|
|
reporter7 = TestReporter(
|
|
max_progress_rows=4,
|
|
mode="min",
|
|
metric="nested/metric_2",
|
|
sort_by_metric=True,
|
|
metric_columns=["nested/metric_2"],
|
|
)
|
|
reporter7.report(trials, done=False)
|
|
assert EXPECTED_NESTED_SORT_RESULT in reporter7._output
|
|
|
|
def testEndToEndReporting(self):
|
|
try:
|
|
os.environ["_TEST_TUNE_TRIAL_UUID"] = "xxxxx"
|
|
os.environ["TUNE_MAX_PENDING_TRIALS_PG"] = "100"
|
|
output = run_string_as_driver(END_TO_END_COMMAND)
|
|
try:
|
|
# New execution path is too fast, trials are already terminated
|
|
if os.environ.get("TUNE_NEW_EXECUTION") == "0":
|
|
assert EXPECTED_END_TO_END_START in output
|
|
assert EXPECTED_END_TO_END_END in output
|
|
for line in output.splitlines():
|
|
if "(raylet)" in line:
|
|
assert "Setting" in line, "Unexpected raylet log messages"
|
|
except Exception:
|
|
print("*** BEGIN OUTPUT ***")
|
|
print(output)
|
|
print("*** END OUTPUT ***")
|
|
raise
|
|
finally:
|
|
del os.environ["_TEST_TUNE_TRIAL_UUID"]
|
|
|
|
def testVerboseReporting(self):
|
|
try:
|
|
os.environ["_TEST_TUNE_TRIAL_UUID"] = "xxxxx"
|
|
|
|
verbose_0_cmd = VERBOSE_CMD + "verbose=0)"
|
|
output = run_string_as_driver(verbose_0_cmd)
|
|
try:
|
|
self.assertNotIn(VERBOSE_EXP_OUT_1, output)
|
|
self.assertNotIn(VERBOSE_EXP_OUT_2, output)
|
|
self.assertNotIn(VERBOSE_TRIAL_NORM_1, output)
|
|
self.assertIsNone(re.search(VERBOSE_TRIAL_NORM_2_PATTERN, output))
|
|
self.assertNotIn(VERBOSE_TRIAL_NORM_3, output)
|
|
self.assertNotIn(VERBOSE_TRIAL_NORM_4, output)
|
|
if os.environ.get("TUNE_NEW_EXECUTION") == "0":
|
|
self.assertNotIn(VERBOSE_TRIAL_DETAIL, output)
|
|
except Exception:
|
|
print("*** BEGIN OUTPUT ***")
|
|
print(output)
|
|
print("*** END OUTPUT ***")
|
|
raise
|
|
|
|
verbose_1_cmd = VERBOSE_CMD + "verbose=1)"
|
|
output = run_string_as_driver(verbose_1_cmd)
|
|
try:
|
|
# New execution path is too fast, trials are already terminated
|
|
if os.environ.get("TUNE_NEW_EXECUTION") == "0":
|
|
self.assertIn(VERBOSE_EXP_OUT_1, output)
|
|
self.assertIn(VERBOSE_EXP_OUT_2, output)
|
|
self.assertNotIn(VERBOSE_TRIAL_NORM_1, output)
|
|
self.assertIsNone(re.search(VERBOSE_TRIAL_NORM_2_PATTERN, output))
|
|
self.assertNotIn(VERBOSE_TRIAL_NORM_3, output)
|
|
self.assertNotIn(VERBOSE_TRIAL_NORM_4, output)
|
|
if os.environ.get("TUNE_NEW_EXECUTION") == "0":
|
|
self.assertNotIn(VERBOSE_TRIAL_DETAIL, output)
|
|
except Exception:
|
|
print("*** BEGIN OUTPUT ***")
|
|
print(output)
|
|
print("*** END OUTPUT ***")
|
|
raise
|
|
|
|
verbose_2_cmd = VERBOSE_CMD + "verbose=2)"
|
|
output = run_string_as_driver(verbose_2_cmd)
|
|
try:
|
|
if os.environ.get("TUNE_NEW_EXECUTION") == "0":
|
|
self.assertIn(VERBOSE_EXP_OUT_1, output)
|
|
self.assertIn(VERBOSE_EXP_OUT_2, output)
|
|
self.assertIn(VERBOSE_TRIAL_NORM_1, output)
|
|
self.assertIsNotNone(re.search(VERBOSE_TRIAL_NORM_2_PATTERN, output))
|
|
self.assertIn(VERBOSE_TRIAL_NORM_3, output)
|
|
self.assertIn(VERBOSE_TRIAL_NORM_4, output)
|
|
self.assertNotIn(VERBOSE_TRIAL_DETAIL, output)
|
|
except Exception:
|
|
print("*** BEGIN OUTPUT ***")
|
|
print(output)
|
|
print("*** END OUTPUT ***")
|
|
raise
|
|
|
|
verbose_3_cmd = VERBOSE_CMD + "verbose=3)"
|
|
output = run_string_as_driver(verbose_3_cmd)
|
|
try:
|
|
if os.environ.get("TUNE_NEW_EXECUTION") == "0":
|
|
self.assertIn(VERBOSE_EXP_OUT_1, output)
|
|
self.assertIn(VERBOSE_EXP_OUT_2, output)
|
|
self.assertNotIn(VERBOSE_TRIAL_NORM_1, output)
|
|
self.assertIsNone(re.search(VERBOSE_TRIAL_NORM_2_PATTERN, output))
|
|
self.assertNotIn(VERBOSE_TRIAL_NORM_3, output)
|
|
self.assertNotIn(VERBOSE_TRIAL_NORM_4, output)
|
|
if os.environ.get("TUNE_NEW_EXECUTION") == "0":
|
|
self.assertIn(VERBOSE_TRIAL_DETAIL, output)
|
|
# Check that we don't print duplicate results at the end
|
|
self.assertTrue(output.count(VERBOSE_TRIAL_WITH_ONCE_RESULT) == 1)
|
|
self.assertIn(VERBOSE_TRIAL_WITH_ONCE_COMPLETED, output)
|
|
except Exception:
|
|
print("*** BEGIN OUTPUT ***")
|
|
print(output)
|
|
print("*** END OUTPUT ***")
|
|
raise
|
|
finally:
|
|
del os.environ["_TEST_TUNE_TRIAL_UUID"]
|
|
|
|
def testReporterDetection(self):
|
|
"""Test if correct reporter is returned from ``detect_reporter()``"""
|
|
reporter = _detect_reporter()
|
|
self.assertTrue(isinstance(reporter, CLIReporter))
|
|
self.assertFalse(isinstance(reporter, JupyterNotebookReporter))
|
|
|
|
with patch("ray.tune.progress_reporter.IS_NOTEBOOK", True):
|
|
reporter = _detect_reporter()
|
|
self.assertFalse(isinstance(reporter, CLIReporter))
|
|
self.assertTrue(isinstance(reporter, JupyterNotebookReporter))
|
|
trainer_reporter = _detect_reporter(_trainer_api=True)
|
|
self.assertFalse(isinstance(trainer_reporter, JupyterNotebookReporter))
|
|
self.assertTrue(isinstance(trainer_reporter, CLIReporter))
|
|
|
|
def testProgressReporterAPI(self):
|
|
class CustomReporter(ProgressReporter):
|
|
def should_report(self, trials, done=False):
|
|
return True
|
|
|
|
def report(self, trials, done, *sys_info):
|
|
pass
|
|
|
|
tune.run(
|
|
lambda config: 2,
|
|
num_samples=1,
|
|
progress_reporter=CustomReporter(),
|
|
verbose=3,
|
|
)
|
|
|
|
def testMaxLen(self):
|
|
trials = []
|
|
for i in range(5):
|
|
t = Mock()
|
|
t.status = "TERMINATED"
|
|
t.trial_id = "%05d" % i
|
|
t.local_experiment_path = "/foo"
|
|
t.temporary_state = Mock()
|
|
t.temporary_state.location = "here"
|
|
t.config = {"verylong" * 20: i}
|
|
t.evaluated_params = {"verylong" * 20: i}
|
|
t.last_result = {"some_metric": "evenlonger" * 100}
|
|
t.__str__ = lambda self: self.trial_id
|
|
trials.append(t)
|
|
|
|
progress_str = _trial_progress_str(
|
|
trials, metric_columns=["some_metric"], force_table=True
|
|
)
|
|
assert any(len(row) <= 90 for row in progress_str.split("\n"))
|
|
|
|
|
|
def test_max_len():
|
|
assert (
|
|
_max_len("some_long_string/even_longer", max_len=28)
|
|
== "some_long_string/even_longer"
|
|
)
|
|
assert _max_len("some_long_string/even_longer", max_len=15) == ".../even_longer"
|
|
|
|
assert (
|
|
_max_len(
|
|
"19_character_string/19_character_string/too_long", max_len=20, wrap=True
|
|
)
|
|
== "...r_string/19_chara\ncter_string/too_long"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
|
|
sys.exit(pytest.main(["-v", __file__]))
|