chore: import upstream snapshot with attribution

test/utils/README.md

@@ -0,0 +1,116 @@
+# 🐳 Run Docker & Qlib
+---
+
+## 📄 Description
+This guide explains how to run the Qlib Docker test file located at `test/utils/test_env.py` in the RD-Agent repository.
+
+---
+
+## 🚀 Running Instructions
+
+### 1. Install the required Python libraries
+- Ensure that the `docker` Python library is installed:
+    ```sh
+    pip install docker
+    ```
+
+### 2. Run the test script
+- Execute the test script to verify the Docker environment setup:
+    ```sh
+    python test/utils/test_env.py
+    ```
+
+### Troubleshooting
+- **PermissionError: [Errno 13] Permission denied.**
+    > This error occurs when the current user does not have the necessary permissions to access the Docker socket. To resolve this issue, follow these steps:
+
+1. **Add the current user to the `docker` group**
+Docker requires root or `docker` group user permissions to access the Docker socket. Add the current user to the `docker` group:
+    ```sh
+    sudo usermod -aG docker $USER
+    ```
+
+2. **Refresh group changes**
+To apply the group changes, log out and log back in, or use the following command:
+    ```sh
+    newgrp docker
+    ```
+
+3. **Verify Docker access**
+Run the following command to ensure that Docker can be accessed:
+    ```sh
+    docker run hello-world
+    ```
+
+4. **Rerun the test script**
+    After completing these steps, rerun the test script:
+    ```sh
+    python test/utils/test_env.py
+    ```
+---
+## 🛠️ Detailed Qlib Docker Function Framework
+
+Here, we provide an overview of the specific functions within the Qlib Docker framework, their purposes, and examples of how to call them.
+
+### QTDockerEnv Class in `env.py`
+
+The `QTDockerEnv` class is responsible for setting up and running Docker environments for Qlib experiments. 
+
+#### Methods:
+
+1. **prepare()**
+   - **Purpose**: Prepares the Docker environment for running experiments. This includes building the Docker image if necessary.
+   - **Example**:
+     ```python
+     qtde = QTDockerEnv()
+     qtde.prepare()
+     ```
+
+2. **run(local_path: str, entry: str) -> str**
+   - **Purpose**: Runs a specified entry point (e.g., a configuration file) in the prepared Docker environment.
+   - **Parameters**:
+     - `local_path`: Path to the local directory to mount into the Docker container.
+     - `entry`: Command or entry point to run inside the Docker container.
+   - **Returns**: The stdout output from the Docker container.
+   - **Example**:
+     ```python
+     result = qtde.run(local_path="/path/to/env_tpl", entry="qrun conf.yaml")
+     ```
+---
+### 📊 Expected Output
+
+Upon successful execution, the test script will produce analysis results of benchmark returns and various risk metrics. The expected output should be similar to:
+
+```
+'The following are analysis results of benchmark return (1 day).'
+risk
+mean               0.000477
+std                0.012295
+annualized_return  0.113561
+information_ratio  0.598699
+max_drawdown      -0.370479
+
+'The following are analysis results of the excess return without cost (1 day).'
+risk
+mean               0.000530
+std                0.005718
+annualized_return  0.126029
+information_ratio  1.428574
+max_drawdown      -0.072310
+
+'The following are analysis results of the excess return with cost (1 day).'
+risk
+mean               0.000339
+std                0.005717
+annualized_return  0.080654
+information_ratio  0.914486
+max_drawdown      -0.086083
+
+'The following are analysis results of indicators (1 day).'
+value
+ffr    1.0
+pa     0.0
+pos    0.0
+```
+
+By following these steps and using the provided functions, you should be able to run the Qlib Docker tests and obtain the expected analysis results.

test/utils/coder/test_CoSTEER.py

@@ -0,0 +1,54 @@
+import unittest
+
+
+class CoSTEERTest(unittest.TestCase):
+
+    def setUp(self):
+        self.test_competition = "aerial-cactus-identification"
+
+    def tearDown(self):
+        pass
+
+    def to_str(self, obj):
+        return "".join(str(obj).split())
+
+    def test_data_loader(self):
+        from rdagent.components.coder.data_science.raw_data_loader.test import (
+            develop_one_competition,
+        )
+
+        # if all tasks in exp are failed, will raise CoderError
+        exp = develop_one_competition(self.test_competition)
+
+    def test_feature(self):
+        from rdagent.components.coder.data_science.feature.test import (
+            develop_one_competition,
+        )
+
+        exp = develop_one_competition(self.test_competition)
+
+    def test_model(self):
+        from rdagent.components.coder.data_science.model.test import (
+            develop_one_competition,
+        )
+
+        exp = develop_one_competition(self.test_competition)
+
+    def test_ensemble(self):
+        from rdagent.components.coder.data_science.ensemble.test import (
+            develop_one_competition,
+        )
+
+        exp = develop_one_competition(self.test_competition)
+
+    def test_workflow(self):
+        from rdagent.components.coder.data_science.workflow.test import (
+            develop_one_competition,
+        )
+
+        exp = develop_one_competition(self.test_competition)
+
+
+if __name__ == "__main__":
+    unittest.main()
+    # pytest test/utils/coder/test_CoSTEER.py

test/utils/test_agent_infra.py

@@ -0,0 +1,35 @@
+import unittest
+
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+
+class TestAgentInfra(unittest.TestCase):
+    def test_agent_infra(self):
+        # NOTE: It is not serious. It is just for testing
+        sys_prompt = T("components.proposal.prompts:hypothesis_gen.system_prompt").r(
+            targets="targets",
+            scenario=T("scenarios.qlib.experiment.prompts:qlib_model_background").r(),
+            hypothesis_output_format=PythonAgentOut.get_spec(),
+            hypothesis_specification=PythonAgentOut.get_spec(),
+        )
+        user_prompt = T("components.proposal.prompts:hypothesis_gen.user_prompt").r(
+            hypothesis_and_feedback="No Feedback",
+            RAG="No RAG",
+            targets="targets",
+        )
+        resp = APIBackend().build_messages_and_create_chat_completion(user_prompt=user_prompt, system_prompt=sys_prompt)
+        code = PythonAgentOut.extract_output(resp)
+
+        print(code)
+
+    def test_include(self):
+        parent = T("components.coder.data_science.raw_data_loader.prompts:spec.user.data_loader").r(latest_spec=None)
+        child = T("scenarios.data_science.share:component_spec.DataLoadSpec").r()
+        assert child in parent
+        print(parent)
+
+
+if __name__ == "__main__":
+    unittest.main()

test/utils/test_conf.py

@@ -0,0 +1,45 @@
+import os
+import unittest
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.scenarios.data_science.dev.runner import DSRunnerCoSTEERSettings
+from rdagent.utils.env import EnvConf, QlibDockerConf
+
+
+class ConfUtils(unittest.TestCase):
+
+    def test_conf(self):
+
+        os.environ["MEM_LIMIT"] = "200g"
+        os.environ["RUNNING_TIMEOUT_PERIOD"] = "None"
+        assert QlibDockerConf().mem_limit == "200g"  # base class will affect subclasses
+        os.environ["QLIB_DOCKER_MEM_LIMIT"] = "300g"
+        assert QlibDockerConf().mem_limit == "300g"  # more accurate subclass will override the base class
+        assert QlibDockerConf().running_timeout_period is None
+
+        os.environ["DEFAULT_ENTRY"] = "which python"
+        os.environ["ENABLE_CACHE"] = "False"
+
+        assert EnvConf().enable_cache is False
+        assert QlibDockerConf().enable_cache is False
+
+        os.environ["ENABLE_CACHE"] = "True"
+        assert EnvConf().enable_cache is True
+        assert QlibDockerConf().enable_cache is True
+
+    def test_ds_costeer_conf(self):
+        os.environ["DS_CODER_COSTEER_MAX_SECONDS_MULTIPLIER"] = "1000"
+        coder_conf = DSCoderCoSTEERSettings()
+        runner_conf = DSRunnerCoSTEERSettings()
+        print(coder_conf.max_seconds_multiplier)
+        print(runner_conf.max_seconds_multiplier)
+        assert coder_conf.max_seconds_multiplier == 1000
+        # NOTE: coder's config should not affect runner's config
+        assert runner_conf.max_seconds_multiplier == 1
+        os.environ["DS_RUNNER_COSTEER_MAX_SECONDS"] = "2000"
+        assert DSRunnerCoSTEERSettings().max_seconds == 2000
+
+
+if __name__ == "__main__":
+    unittest.main()

test/utils/test_env.py

@@ -0,0 +1,160 @@
+import os
+import sys
+import time
+import unittest
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parent.parent))
+import shutil
+
+from rdagent.utils.env import (
+    CondaConf,
+    LocalConf,
+    LocalEnv,
+    QlibDockerConf,
+    QTDockerEnv,
+    cleanup_container,
+)
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class QlibLocalEnv(LocalEnv):
+    def prepare(self) -> None:
+        if not (Path("~/.qlib/qlib_data/cn_data").expanduser().resolve().exists()):
+            self.check_output(
+                entry="python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn",
+            )
+        else:
+            print("Data already exists. Download skipped.")
+
+
+class EnvUtils(unittest.TestCase):
+    def setUp(self):
+        self.test_workspace = DIRNAME / "test_workspace"
+        self.test_workspace.mkdir(exist_ok=True)
+
+    def tearDown(self):
+        if self.test_workspace.exists():
+            shutil.rmtree(self.test_workspace)
+
+    # NOTE: Since I don't know the exact environment in which it will be used, here's just an example.
+    # NOTE: Because you need to download the data during the prepare process. So you need to have pyqlib in your environment.
+    def test_local(self):
+        local_conf = LocalConf(
+            bin_path="/home/v-linlanglv/miniconda3/envs/RD-Agent-310/bin",
+            default_entry="qrun conf.yaml",
+        )
+        qle = QlibLocalEnv(conf=local_conf)
+        qle.prepare()
+        conf_path = str(DIRNAME / "env_tpl" / "conf.yaml")
+        qle.check_output(entry="qrun " + conf_path)
+        mlrun_p = DIRNAME / "env_tpl" / "mlruns"
+        self.assertTrue(mlrun_p.exists(), f"Expected output file {mlrun_p} not found")
+
+    def test_local_simple(self):
+        code_path = DIRNAME / "tmp_code"
+        code_path.mkdir(exist_ok=True)
+        # Get user home dynamically
+        home_bin = str(Path.home() / "miniconda3/bin/")
+        local_conf = LocalConf(bin_path=home_bin, default_entry="which python")
+
+        local_conf.extra_volumes = {str(code_path): "./code"}
+        print(local_conf)
+        le = LocalEnv(conf=local_conf)
+        le.prepare()
+        result = le.run(local_path=str(code_path))
+        print(result.stdout, result.exit_code, result.running_time)
+
+    def test_conda_simple(self):
+        conda_conf = CondaConf(default_entry="which python", conda_env_name="MLE")
+        le = LocalEnv(conf=conda_conf)
+        le.prepare()
+        code_path = DIRNAME / "tmp_code"
+        code_path.mkdir(exist_ok=True)
+        result = le.run(local_path=str(code_path))
+        print(result.stdout, result.exit_code, result.running_time)
+
+    def test_conda_error(self):
+        conda_conf = CondaConf(conda_env_name="MLE")
+        le = LocalEnv(conf=conda_conf)
+        le.prepare()
+        file_name = f"{time.time()}.py"
+        with open(self.test_workspace / file_name, "w") as f:
+            f.write('import json \njson.loads(b\'{"name": "\xa1"}\')')
+        result = le.run(local_path=str(self.test_workspace), entry=f"python {file_name}")
+        assert result.exit_code == 1
+        assert "bytes can only contain ASCII literal characters" in result.stdout
+
+    def test_docker(self):
+        """We will mount `env_tpl` into the docker image.
+        And run the docker image with `qrun conf.yaml`
+        """
+        qtde = QTDockerEnv()
+        qtde.prepare()  # you can prepare for multiple times. It is expected to handle it correctly
+        # qtde.run("nvidia-smi")  # NOTE: you can check your GPU with this command
+        # the stdout are returned as result
+        result = qtde.check_output(local_path=str(DIRNAME / "env_tpl"), entry="qrun conf.yaml")
+
+        mlrun_p = DIRNAME / "env_tpl" / "mlruns"
+        self.assertTrue(mlrun_p.exists(), f"Expected output file {mlrun_p} not found")
+
+        # read experiment
+        result = qtde.check_output(local_path=str(DIRNAME / "env_tpl"), entry="python read_exp_res.py")
+        print(result)
+
+    def test_run(self):
+        """Test the run method of QTDockerEnv with both valid and invalid commands."""
+        qtde = QTDockerEnv()
+        qtde.prepare()
+
+        # Test with a valid command
+        result = qtde.run(entry='echo "Hello, World!"', local_path=str(self.test_workspace))
+        print(result.exit_code)
+        assert result.exit_code == 0, f"Expected return code 0, but got {result.exit_code}"
+        assert "Hello, World!" in result.stdout, "Expected output not found in result"
+
+        # Test with an invalid command
+        result = qtde.run(entry="invalid_command", local_path=str(self.test_workspace))
+        print(result.exit_code)
+        assert result.exit_code != 0, "Expected non-zero return code for invalid command"
+
+        dc = QlibDockerConf()
+        dc.running_timeout_period = 1
+        qtde = QTDockerEnv(dc)
+        result = qtde.run(entry="sleep 2", local_path=str(self.test_workspace))
+        print(result.exit_code)
+        assert result.exit_code == 124, "Expected return code 124 for timeout"
+
+    def test_docker_mem(self):
+        cmd = 'python -c \'print("start"); import numpy as np;  size_mb = 500; size = size_mb * 1024 * 1024 // 8; array = np.random.randn(size).astype(np.float64); print("success")\''
+
+        qtde = QTDockerEnv(QlibDockerConf(mem_limit="10m"))
+        qtde.prepare()
+        result = qtde.check_output(local_path=str(DIRNAME / "env_tpl"), entry=cmd)
+        self.assertTrue(not result.strip().endswith("success"))
+
+        qtde = QTDockerEnv(QlibDockerConf(mem_limit="1g"))
+        qtde.prepare()
+        result = qtde.check_output(local_path=str(DIRNAME / "env_tpl"), entry=cmd)
+        self.assertTrue(result.strip().endswith("success"))
+
+        # The above command equals to the follow commands with dockr cli.sh
+        # docker run  --memory=10m  -it --rm local_qlib:latest python -c 'import numpy as np; print(123);  size_mb = 1; size = size_mb * 1024 * 1024 // 8; array = np.random.randn(size).astype(np.float64); array[0], array[-1] = 1.0, 1.0; print(321)'
+        # docker run  --memory=10g  -it --rm local_qlib:latest python -c 'import numpy as np; print(123);  size_mb = 1; size = size_mb * 1024 * 1024 // 8; array = np.random.randn(size).astype(np.float64); array[0], array[-1] = 1.0, 1.0; print(321)'
+
+    def test_cleanup_container_import(self):
+        """Test that cleanup_container function can be imported and has correct interface."""
+        # Test that the function exists and can be called
+        self.assertTrue(callable(cleanup_container))
+
+        # Test with None (should not raise an exception)
+        cleanup_container(None, "test context")
+
+        # The function should accept positional and keyword arguments
+        cleanup_container(None)
+        cleanup_container(None, context="test")
+
+
+if __name__ == "__main__":
+    unittest.main()

test/utils/test_import.py

@@ -0,0 +1,61 @@
+import importlib
+import os
+import unittest
+from pathlib import Path
+
+import pytest
+
+
+@pytest.mark.offline
+class TestRDAgentImports(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.rdagent_directory = Path(__file__).resolve().parent.parent.parent
+        cls.modules = list(cls.import_all_modules_from_directory(cls.rdagent_directory))
+
+    @staticmethod
+    def import_all_modules_from_directory(directory):
+        for file in directory.joinpath("rdagent").rglob("*.py"):
+            fstr = str(file)
+            if "example" in fstr:
+                continue
+            if "meta_tpl" in fstr:
+                continue
+            if "autorl_bench/workspace/" in fstr:
+                continue
+            if "template" in fstr or "tpl" in fstr:
+                continue
+            if "model_coder" in fstr:
+                continue
+            if "llm_st" in fstr:
+                continue
+            if (
+                "rdagent/log/ui/" in fstr
+                or fstr.endswith("rdagent/app/cli.py")
+                or fstr.endswith("rdagent/app/CI/run.py")
+                or fstr.endswith("rdagent/app/utils/ape.py")
+                or fstr.endswith("rdagent/log/ui/utils.py")
+            ):
+                # the entrance points
+                continue
+            # llamafactory==0.9.3 pins numpy to an older version, causing other
+            # installations to fail. The `extract_parameters` tests are therefore
+            # temporarily disabled and can be re-enabled once the numpy constraint is relaxed.
+            if "extract_parameters" in fstr:
+                continue
+
+            yield fstr[fstr.index("rdagent") : -3].replace("/", ".")
+
+    def test_import_modules(self):
+        print(self.modules)
+        for module_name in self.modules:
+            with self.subTest(module=module_name):
+                try:
+                    print(module_name)
+                    importlib.import_module(module_name)
+                except Exception as e:
+                    self.fail(f"Failed to import {module_name}: {e}")
+
+
+if __name__ == "__main__":
+    unittest.main()

test/utils/test_kaggle.py

@@ -0,0 +1,32 @@
+import unittest
+from pathlib import Path
+
+from rich import print
+
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
+from rdagent.scenarios.kaggle.experiment.workspace import KGFBWorkspace
+from rdagent.scenarios.kaggle.kaggle_crawler import download_data
+
+
+class TestTpl(unittest.TestCase):
+    def test_competition_template(self):
+        """
+        export KG_COMPETITION=<competition_name> before running this test
+        """
+        competition = KAGGLE_IMPLEMENT_SETTING.competition
+        print(f"[bold orange]{competition}[/bold orange]")
+        download_data(competition, settings=KAGGLE_IMPLEMENT_SETTING)
+        ws = KGFBWorkspace(
+            template_folder_path=Path(__file__).parent.parent.parent
+            / KAGGLE_IMPLEMENT_SETTING.template_path
+            / f"{competition}",
+        )
+        print(ws.workspace_path)
+        ws.execute()
+        success = (ws.workspace_path / "submission.csv").exists()
+        self.assertTrue(success, "submission.csv is not generated")
+        # ws.clear()
+
+
+if __name__ == "__main__":
+    unittest.main()

test/utils/test_misc.py

@@ -0,0 +1,74 @@
+import unittest
+
+import pytest
+
+from rdagent.core.utils import SingletonBaseClass
+
+
+class A(SingletonBaseClass):
+    def __init__(self, **kwargs):
+        print(self, "__init__", kwargs)  # make sure the __init__ is called only once.
+        self.kwargs = kwargs
+
+    def __str__(self) -> str:
+        return f"{self.__class__.__name__}.{getattr(self, 'kwargs', None)}"
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+
+@pytest.mark.offline
+class MiscTest(unittest.TestCase):
+    def test_singleton(self):
+        print("a1=================")
+        a1 = A()
+        print("a2=================")
+        a2 = A()
+        print("a3=================")
+        a3 = A(x=3)
+        print("a4=================")
+        a4 = A(x=2)
+        print("a5=================")
+        a5 = A(b=3)
+        print("a6=================")
+        a6 = A(x=3)
+
+        # Check that a1 and a2 are the same instance
+        self.assertIs(a1, a2)
+
+        # Check that a3 and a6 are the same instance
+        self.assertIs(a3, a6)
+
+        # Check that a1 and a3 are different instances
+        self.assertIsNot(a1, a3)
+
+        # Check that a3 and a4 are different instances
+        self.assertIsNot(a3, a4)
+
+        # Check that a4 and a5 are different instances
+        self.assertIsNot(a4, a5)
+
+        # Check that a5 and a6 are different instances
+        self.assertIsNot(a5, a6)
+
+        print(id(a1), id(a2), id(a3), id(a4), id(a5), id(a6))
+
+        print("...................... Start testing pickle ......................")
+
+        # Test pickle
+        import pickle
+
+        with self.assertRaises(pickle.PicklingError):
+            with open("a3.pkl", "wb") as f:
+                pickle.dump(a3, f)
+        # NOTE: If the pickle feature is not disabled,
+        # loading a3.pkl will return a1, and a1 will be updated with a3's attributes.
+        # print(a1.kwargs)
+        # with open("a3.pkl", "rb") as f:
+        #     a3_pkl = pickle.load(f)
+        # print(id(a3), id(a3_pkl))  # not the same object
+        # print(a1.kwargs)  # a1 will be changed.
+
+
+if __name__ == "__main__":
+    unittest.main()

test/utils/test_ws.py

@@ -0,0 +1,76 @@
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from rdagent.core.experiment import FBWorkspace
+
+
+class TestFBWorkspace(unittest.TestCase):
+    """
+    Unit-tests for `FBWorkspace`.
+    """
+
+    def setUp(self) -> None:  # noqa: D401
+        """
+        Create an isolated temporary directory for each test case.
+        """
+        self._tmp_dir = tempfile.TemporaryDirectory()
+        self.tmp_path = Path(self._tmp_dir.name)
+
+    def tearDown(self) -> None:
+        """
+        Clean up the temporary directory created in :py:meth:`setUp`.
+        """
+        self._tmp_dir.cleanup()
+
+    def test_checkpoint_roundtrip(self) -> None:
+        """
+        Verify that ``create_ws_ckp`` captures the current workspace state and
+        ``recover_ws_ckp`` faithfully restores it.
+        """
+        # create a symbolic link inside workspace and ensure checkpoint preserves the link
+        external_file = self.tmp_path / "external.txt"
+        external_file.write_text("external data")
+        ws = FBWorkspace()
+        ws.workspace_path = self.tmp_path / "ws"
+        ws.prepare()
+        (ws.workspace_path / "sym.txt").symlink_to(external_file)
+        ws.inject_files(**{"foo.py": "print('hi')", "bar.py": "x = 1"})
+
+        # Snapshot current workspace
+        original_files = {
+            p.relative_to(ws.workspace_path): (os.readlink(p) if p.is_symlink() else p.read_text())
+            for p in ws.workspace_path.rglob("*")
+            if p.is_file() or p.is_symlink()
+        }
+        ws.create_ws_ckp()
+        self.assertIsNotNone(ws.ws_ckp, "Checkpoint data should have been generated")
+
+        # Mutate workspace
+        (ws.workspace_path / "foo.py").write_text("print('changed')")
+        (ws.workspace_path / "new.py").write_text("pass")
+        (ws.workspace_path / "sym.txt").unlink()
+
+        # Restore and verify equality with snapshot
+        ws.recover_ws_ckp()
+
+        # Ensure symbolic link still exists after recovery.
+        self.assertTrue((ws.workspace_path / "sym.txt").is_symlink())
+        recovered_files = {
+            p.relative_to(ws.workspace_path): (os.readlink(p) if p.is_symlink() else p.read_text())
+            for p in ws.workspace_path.rglob("*")
+            if p.is_file() or p.is_symlink()
+        }
+        self.assertEqual(recovered_files, original_files)
+
+        # Verify large files (>100 KB) are excluded when a size-limit is configured.
+        from rdagent.core.conf import RD_AGENT_SETTINGS as _SETTINGS
+
+        _SETTINGS.workspace_ckp_size_limit = 100 * 1024  # set limit temporarily for this test
+
+        large_file = ws.workspace_path / "large.bin"
+        large_file.write_bytes(b"0" * (110 * 1024))  # 110 KB dummy content
+        ws.create_ws_ckp()
+        ws.recover_ws_ckp()
+        self.assertFalse((ws.workspace_path / "large.bin").exists())