chore: import upstream snapshot with attribution

python/ray/tune/tests/test_placeholder.py

@@ -0,0 +1,316 @@
+import unittest
+
+import numpy as np
+
+from ray import tune
+from ray.tune.impl.placeholder import (
+    _FunctionResolver,
+    _RefResolver,
+    create_resolvers_map,
+    inject_placeholders,
+    resolve_placeholders,
+)
+from ray.tune.search.sample import Float, Integer
+
+
+class Dummy:
+    def __init__(self, value):
+        self.value = value
+
+
+class PlaceholderTest(unittest.TestCase):
+    def testNotReplaced(self):
+        config = {
+            "param1": "ok",
+            "param2": ["not ok", tune.grid_search(["ok", "not ok"])],
+            "param3": {
+                "param4": tune.choice(["ok", "not ok"]),
+            },
+        }
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        # Primitive typed choices are not replaced.
+        self.assertEqual(config["param2"][1]["grid_search"], ["ok", "not ok"])
+        self.assertEqual(config["param3"]["param4"].categories, ["ok", "not ok"])
+
+    def testGridSearch(self):
+        config = {
+            "param1": "ok",
+            "param2": ["not ok", tune.grid_search(["ok", Dummy("not ok")])],
+            "param3": {
+                "param4": tune.grid_search([Dummy("ok"), "not ok"]),
+            },
+        }
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        self.assertEqual(
+            config["param2"][1]["grid_search"],
+            ["ok", (_RefResolver.TOKEN, "1870fa9b")],
+        )
+        self.assertEqual(
+            config["param3"]["param4"]["grid_search"],
+            [(_RefResolver.TOKEN, "8515e998"), "not ok"],
+        )
+
+        # Pretend we picked a choice from the grid searches.
+        config["param2"][1] = (_RefResolver.TOKEN, "1870fa9b")
+        config["param3"]["param4"] = "not ok"
+
+        resolve_placeholders(config, replaced)
+
+        self.assertEqual(config["param2"][1].value, "not ok")
+        self.assertEqual(config["param3"]["param4"], "not ok")
+
+    def testCategorical(self):
+        config = {
+            "param1": "ok",
+            "param2": ["not ok", tune.choice([Dummy("ok"), "not ok"])],
+            "param3": {
+                "param4": tune.choice([Dummy("ok"), "not ok"]),
+            },
+        }
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        self.assertEqual(
+            config["param2"][1].categories,
+            [(_RefResolver.TOKEN, "ec0e030c"), "not ok"],
+        )
+        self.assertEqual(
+            config["param3"]["param4"].categories,
+            [(_RefResolver.TOKEN, "8515e998"), "not ok"],
+        )
+
+        # Pretend we picked a choice from the categoricals.
+        config["param2"][1] = (_RefResolver.TOKEN, "ec0e030c")
+        config["param3"]["param4"] = "not ok"
+
+        resolve_placeholders(config, replaced)
+
+        self.assertEqual(config["param2"][1].value, "ok")
+        self.assertEqual(config["param3"]["param4"], "not ok")
+
+    def _testNonSearchSpaceRef(self, value):
+        """Tests that non-primitives (numpy, lambda fn) get replaced by a reference."""
+        config = {"param": tune.choice([value, "other", value])}
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        self.assertEqual(
+            config["param"].categories,
+            [
+                (_RefResolver.TOKEN, "ec15c422"),
+                "other",
+                (_RefResolver.TOKEN, "3c7edff5"),
+            ],
+        )
+
+    def testNumpyToRef(self):
+        self._testNonSearchSpaceRef(np.arange(10))
+
+    def testLambdaToRef(self):
+        self._testNonSearchSpaceRef(lambda x: x)
+
+    def testFunction(self):
+        config = {
+            "param1": "ok",
+            "param2": ["not ok", tune.sample_from(lambda: "not ok")],
+            # Both lambdas, either taking spec or config, should work.
+            "param3": {
+                "param4": tune.sample_from(lambda spec: spec["config"]["param1"]),
+            },
+            "param4": {
+                "param4": tune.sample_from(lambda config: config["param1"]),
+            },
+            # Make sure dot notation also works with spec passed in.
+            "param5": {
+                "param4": tune.sample_from(lambda spec: spec.config["param1"]),
+            },
+        }
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        self.assertEqual(config["param2"][1][0], _FunctionResolver.TOKEN)
+        self.assertEqual(config["param3"]["param4"][0], _FunctionResolver.TOKEN)
+        self.assertEqual(config["param4"]["param4"][0], _FunctionResolver.TOKEN)
+        self.assertEqual(config["param5"]["param4"][0], _FunctionResolver.TOKEN)
+
+        resolve_placeholders(config, replaced)
+
+        self.assertEqual(config["param2"][1], "not ok")
+        self.assertEqual(config["param3"]["param4"], "ok")
+        self.assertEqual(config["param4"]["param4"], "ok")
+        self.assertEqual(config["param5"]["param4"], "ok")
+
+    def testRefValue(self):
+        config = {
+            "param1": "ok",
+            "param2": ["not ok", Dummy("ok")],
+            "param3": {
+                "param4": Dummy("not ok"),
+            },
+        }
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        self.assertEqual(config["param2"][1][0], _RefResolver.TOKEN)
+        self.assertEqual(config["param3"]["param4"][0], _RefResolver.TOKEN)
+
+        resolve_placeholders(config, replaced)
+
+        self.assertEqual(config["param2"][1].value, "ok")
+        self.assertEqual(config["param3"]["param4"].value, "not ok")
+
+    def testTuple(self):
+        class Dummy:
+            def __init__(self, value):
+                self.value = value
+
+        config = {
+            "param1": ("ok", "not ok"),
+            "param2": ["not ok", (1, Dummy("ok"))],
+            "param3": {
+                "param4": (1, [2, Dummy("not ok")], 3),
+            },
+        }
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        self.assertTrue(isinstance(config["param1"], tuple))
+        self.assertEqual(config["param1"], ("ok", "not ok"))
+        self.assertTrue(isinstance(config["param2"][1], tuple))
+        self.assertTrue(isinstance(config["param3"]["param4"], tuple))
+
+        resolve_placeholders(config, replaced)
+
+        self.assertTrue(isinstance(config["param2"][1], tuple))
+        self.assertEqual(config["param2"][1][1].value, "ok")
+        self.assertTrue(isinstance(config["param3"]["param4"], tuple))
+        self.assertEqual(config["param3"]["param4"][1][1].value, "not ok")
+
+    def testOtherDomains(self):
+        config = {
+            "param1": tune.uniform(0, 1),
+            "param2": tune.randint(2, 3),
+            "param3": tune.qrandn(0, 1, 0.1),
+        }
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        # Normal params are not replaced.
+        self.assertTrue(isinstance(config["param1"], Float))
+        self.assertTrue(isinstance(config["param2"], Integer))
+        self.assertTrue(isinstance(config["param3"], Float))
+
+    def testPointToEval(self):
+        config = {
+            "param1": "ok",
+            "param2": ["not ok", tune.choice([Dummy("ok"), "not ok"])],
+            "param3": {
+                "param4": tune.sample_from(lambda spec: spec["config"]["param1"]),
+            },
+        }
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        # Normal params are not replaced.
+        self.assertEqual(
+            config["param2"][1].categories,
+            [(_RefResolver.TOKEN, "ec0e030c"), "not ok"],
+        )
+        self.assertEqual(
+            config["param3"]["param4"], (_FunctionResolver.TOKEN, "134aff3a")
+        )
+
+        # Now, say we manually resolved the placeholders based on
+        # points_to_evaluate.
+        config["param2"][1] = "not_ok"
+        config["param3"]["param4"] = "ok"
+
+        resolve_placeholders(config, replaced)
+
+        # Params stays the same.
+        self.assertEqual(config["param2"][1], "not_ok")
+        self.assertEqual(config["param3"]["param4"], "ok")
+
+    def testSimpleNestedSearchSpaces(self):
+        config = {
+            "param1": "ok",
+            "param2": tune.choice(
+                [
+                    tune.choice([Dummy(1), 2, 3]),
+                    tune.uniform(5, 6),
+                ]
+            ),
+        }
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        # Manually resolve. Select the Dummy value.
+        config["param2"] = (_RefResolver.TOKEN, "6f33af83")
+
+        resolve_placeholders(config, replaced)
+
+        self.assertEqual(config["param2"].value, 1)
+
+    def testSimpleNestedSearchSpaces2(self):
+        config = {
+            "param1": "ok",
+            "param2": tune.choice(
+                [
+                    (None, Dummy(1), None),
+                    (Dummy(2), None, None),
+                    (None, None, Dummy(3)),
+                ]
+            ),
+        }
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        # Manually resolve. Select the Dummy value.
+        config["param2"] = (None, None, (_RefResolver.TOKEN, "a1433750"))
+
+        resolve_placeholders(config, replaced)
+
+        self.assertEqual(config["param2"][2].value, 3)
+
+    def testResolveFunctionAfterRef(self):
+        config = {
+            "param1": "ok",
+            "param2": tune.choice([Dummy("ok"), "not ok"]),
+            "param3": {
+                "param4": tune.sample_from(lambda config: config["param2"]),
+            },
+        }
+
+        replaced = create_resolvers_map()
+        config = inject_placeholders(config, replaced)
+
+        # Manually resolve param2.
+        config["param2"] = (_RefResolver.TOKEN, "07cb6238")
+
+        resolve_placeholders(config, replaced)
+
+        # param3.param4 should get the same value as resolved param2.
+        self.assertEqual(config["param3"]["param4"].value, "ok")
+
+
+if __name__ == "__main__":
+    import sys
+
+    import pytest
+
+    sys.exit(pytest.main(["-v", __file__]))