chore: import upstream snapshot with attribution

rllib/examples/algorithms/ppo/atari_ppo.py

@@ -0,0 +1,97 @@
+# These tags allow extracting portions of this script on Anyscale.
+# ws-template-imports-start
+import gymnasium as gym
+
+from ray import tune
+from ray.rllib.algorithms.ppo import PPOConfig
+from ray.rllib.connectors.env_to_module.frame_stacking import FrameStackingEnvToModule
+from ray.rllib.connectors.learner.frame_stacking import FrameStackingLearner
+from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
+from ray.rllib.env.wrappers.atari_wrappers import wrap_atari_for_new_api_stack
+from ray.rllib.examples.utils import (
+    add_rllib_example_script_args,
+    run_rllib_example_script_experiment,
+)
+
+# ws-template-imports-end
+
+parser = add_rllib_example_script_args(
+    default_reward=float("inf"),
+    default_timesteps=3000000,
+    default_iters=100000000000,
+)
+parser.set_defaults(
+    env="ale_py:ALE/Pong-v5",
+)
+# Use `parser` to add your own custom command line options to this script
+# and (if needed) use their values to set up `config` below.
+args = parser.parse_args()
+
+NUM_LEARNERS = args.num_learners or 1
+ENV = args.env
+
+
+# These tags allow extracting portions of this script on Anyscale.
+# ws-template-code-start
+def _make_env_to_module_connector(env, spaces, device):
+    return FrameStackingEnvToModule(num_frames=4)
+
+
+def _make_learner_connector(input_observation_space, input_action_space):
+    return FrameStackingLearner(num_frames=4)
+
+
+# Create a custom Atari setup (w/o the usual RLlib-hard-coded framestacking in it).
+# We would like our frame stacking connector to do this job.
+def _env_creator(cfg):
+    return wrap_atari_for_new_api_stack(
+        gym.make(ENV, **cfg, render_mode="rgb_array"),
+        # Perform frame-stacking through ConnectorV2 API.
+        framestack=None,
+    )
+
+
+tune.register_env("env", _env_creator)
+
+config = (
+    PPOConfig()
+    .environment(
+        "env",
+        env_config={
+            # Make analogous to old v4 + NoFrameskip.
+            "frameskip": 1,
+            "full_action_space": False,
+            "repeat_action_probability": 0.0,
+        },
+        clip_rewards=True,
+    )
+    .env_runners(
+        env_to_module_connector=_make_env_to_module_connector,
+    )
+    .training(
+        learner_connector=_make_learner_connector,
+        train_batch_size_per_learner=4000,
+        minibatch_size=128,
+        lambda_=0.95,
+        kl_coeff=0.5,
+        clip_param=0.1,
+        vf_clip_param=10.0,
+        entropy_coeff=0.01,
+        num_epochs=10,
+        lr=0.00015 * NUM_LEARNERS,
+        grad_clip=100.0,
+        grad_clip_by="global_norm",
+    )
+    .rl_module(
+        model_config=DefaultModelConfig(
+            conv_filters=[[16, 4, 2], [32, 4, 2], [64, 4, 2], [128, 4, 2]],
+            conv_activation="relu",
+            head_fcnet_hiddens=[256],
+            vf_share_layers=True,
+        ),
+    )
+)
+# ws-template-code-end
+
+if __name__ == "__main__":
+    run_rllib_example_script_experiment(config, args=args)

rllib/examples/algorithms/ppo/benchmark_ppo_mujoco.py

@@ -0,0 +1,138 @@
+from ray import tune
+from ray.rllib.algorithms.ppo.ppo import PPOConfig
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+from ray.tune import Stopper
+
+# Needs the following packages to be installed on Ubuntu:
+#   sudo apt-get libosmesa-dev
+#   sudo apt-get install patchelf
+#   python -m pip install "gymnasium[mujoco]"
+# Might need to be added to bashsrc:
+#   export MUJOCO_GL=osmesa"
+#   export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/.mujoco/mujoco200/bin"
+
+# See the following links for becnhmark results of other libraries:
+#   Original paper: https://arxiv.org/pdf/1707.06347
+#   CleanRL: https://wandb.ai/openrlbenchmark/openrlbenchmark/reports"
+#       /MuJoCo-CleanRL-s-PPO--VmlldzoxODAwNjkw
+#   AgileRL: https://github.com/AgileRL/AgileRL?tab=readme-ov-file#benchmarks
+benchmark_envs = {
+    "HalfCheetah-v4": {
+        f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 2000,
+        f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 1000000,
+    },
+    "Hopper-v4": {
+        f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 2250,
+        f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 1000000,
+    },
+    "InvertedPendulum-v4": {
+        f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 1000,
+        f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 1000000,
+    },
+    "InvertedDoublePendulum-v4": {
+        f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 8000,
+        f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 1000000,
+    },
+    "Reacher-v4": {
+        f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": -15,
+        f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 1000000,
+    },
+    "Swimmer-v4": {
+        f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 120,
+        f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 1000000,
+    },
+    "Walker2d-v4": {
+        f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 3500,
+        f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 1000000,
+    },
+}
+
+
+# Define a `tune.Stopper` that stops the training if the benchmark is reached
+# or the maximum number of timesteps is exceeded.
+class BenchmarkStopper(Stopper):
+    def __init__(self, benchmark_envs):
+        self.benchmark_envs = benchmark_envs
+
+    def __call__(self, trial_id, result):
+        # Stop training if the mean reward is reached.
+        if (
+            result[ENV_RUNNER_RESULTS][EPISODE_RETURN_MEAN]
+            >= self.benchmark_envs[result["env"]][
+                f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}"
+            ]
+        ):
+            return True
+        # Otherwise check, if the total number of timesteps is exceeded.
+        elif (
+            result[f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}"]
+            >= self.benchmark_envs[result["env"]][f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}"]
+        ):
+            return True
+        # Otherwise continue training.
+        else:
+            return False
+
+    # Note, this needs to implemented b/c the parent class is abstract.
+    def stop_all(self):
+        return False
+
+
+config = (
+    PPOConfig()
+    .environment(env=tune.grid_search(list(benchmark_envs.keys())))
+    .env_runners(
+        # Following the paper.
+        num_env_runners=32,
+        rollout_fragment_length=512,
+    )
+    .learners(
+        # Let's start with a small number of learner workers and
+        # add later a tune grid search for these resources.
+        num_learners=1,
+        num_gpus_per_learner=1,
+    )
+    # TODO (simon): Adjust to new model_config_dict.
+    .training(
+        # Following the paper.
+        lambda_=0.95,
+        lr=0.0003,
+        num_epochs=15,
+        train_batch_size=32 * 512,
+        minibatch_size=4096,
+        vf_loss_coeff=0.01,
+        model={
+            "fcnet_hiddens": [64, 64],
+            "fcnet_activation": "tanh",
+            "vf_share_layers": True,
+        },
+    )
+    .reporting(
+        metrics_num_episodes_for_smoothing=5,
+        min_sample_timesteps_per_iteration=1000,
+    )
+    .evaluation(
+        evaluation_duration="auto",
+        evaluation_interval=1,
+        evaluation_num_env_runners=1,
+        evaluation_parallel_to_training=True,
+        evaluation_config={
+            "explore": True,
+        },
+    )
+)
+
+tuner = tune.Tuner(
+    "PPO",
+    param_space=config,
+    run_config=tune.RunConfig(
+        stop=BenchmarkStopper(benchmark_envs=benchmark_envs),
+        name="benchmark_ppo_mujoco",
+    ),
+)
+
+tuner.fit()

rllib/examples/algorithms/ppo/cartpole_heavy_ppo.py

@@ -0,0 +1,38 @@
+from ray.rllib.algorithms.ppo import PPOConfig
+from ray.rllib.connectors.env_to_module import FlattenObservations
+from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
+from ray.rllib.examples.envs.classes.cartpole_with_large_observation_space import (
+    CartPoleWithLargeObservationSpace,
+)
+from ray.rllib.examples.utils import (
+    add_rllib_example_script_args,
+    run_rllib_example_script_experiment,
+)
+
+parser = add_rllib_example_script_args(default_reward=450.0, default_timesteps=300000)
+# Use `parser` to add your own custom command line options to this script
+# and (if needed) use their values to set up `config` below.
+args = parser.parse_args()
+
+config = (
+    PPOConfig()
+    .environment(CartPoleWithLargeObservationSpace)
+    .env_runners(
+        env_to_module_connector=lambda env, spaces, device: FlattenObservations(),
+        episodes_to_numpy=False,
+    )
+    .training(
+        lr=0.0003,
+        num_epochs=6,
+        vf_loss_coeff=0.01,
+    )
+    .rl_module(
+        model_config=DefaultModelConfig(
+            use_lstm=True,
+            lstm_cell_size=1024,
+        ),
+    )
+)
+
+if __name__ == "__main__":
+    run_rllib_example_script_experiment(config, args)

rllib/examples/algorithms/ppo/cartpole_ppo.py

@@ -0,0 +1,32 @@
+from ray.rllib.algorithms.ppo import PPOConfig
+from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
+from ray.rllib.examples.utils import (
+    add_rllib_example_script_args,
+    run_rllib_example_script_experiment,
+)
+
+parser = add_rllib_example_script_args(default_reward=450.0, default_timesteps=300000)
+# Use `parser` to add your own custom command line options to this script
+# and (if needed) use their values to set up `config` below.
+args = parser.parse_args()
+
+config = (
+    PPOConfig()
+    .environment("CartPole-v1")
+    .training(
+        lr=0.0003,
+        num_epochs=6,
+        vf_loss_coeff=0.01,
+    )
+    .rl_module(
+        model_config=DefaultModelConfig(
+            fcnet_hiddens=[32],
+            fcnet_activation="linear",
+            vf_share_layers=True,
+        ),
+    )
+)
+
+
+if __name__ == "__main__":
+    run_rllib_example_script_experiment(config, args)

rllib/examples/algorithms/ppo/cartpole_truncated_ppo.py

@@ -0,0 +1,55 @@
+import gymnasium as gym
+from gymnasium.wrappers import TimeLimit
+
+from ray.rllib.algorithms.ppo import PPOConfig
+from ray.rllib.examples.utils import (
+    add_rllib_example_script_args,
+    run_rllib_example_script_experiment,
+)
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    EVALUATION_RESULTS,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+from ray.tune.registry import register_env
+
+parser = add_rllib_example_script_args()
+# Use `parser` to add your own custom command line options to this script
+# and (if needed) use their values to set up `config` below.
+args = parser.parse_args()
+
+# For training, use a time-truncated (max. 50 timestep) version of CartPole-v1.
+register_env(
+    "cartpole_truncated",
+    lambda _: TimeLimit(gym.make("CartPole-v1"), max_episode_steps=50),
+)
+
+config = (
+    PPOConfig()
+    .environment("cartpole_truncated")
+    .env_runners(num_envs_per_env_runner=10)
+    .training(
+        lr=0.0003,
+        num_epochs=6,
+        vf_loss_coeff=0.01,
+    )
+    # For evaluation, use the "real" CartPole-v1 env (up to 500 steps).
+    .evaluation(
+        evaluation_config=PPOConfig.overrides(
+            env="CartPole-v1",
+            explore=False,
+        ),
+        evaluation_interval=1,
+        evaluation_num_env_runners=1,
+    )
+)
+
+stop = {
+    f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 500000,
+    f"{EVALUATION_RESULTS}/{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 80.0,
+}
+
+
+if __name__ == "__main__":
+    run_rllib_example_script_experiment(config, args, stop=stop)

rllib/examples/algorithms/ppo/memory_leak_test_ppo_new_stack.py

@@ -0,0 +1,13 @@
+from ray.rllib.algorithms.ppo import PPOConfig
+from ray.rllib.examples.envs.classes.random_env import RandomLargeObsSpaceEnv
+
+config = (
+    PPOConfig()
+    # Switch off np.random, which is known to have memory leaks.
+    .environment(RandomLargeObsSpaceEnv, env_config={"static_samples": True})
+    .env_runners(
+        num_env_runners=4,
+        num_envs_per_env_runner=5,
+    )
+    .training(train_batch_size=500, minibatch_size=256, num_epochs=5)
+)

rllib/examples/algorithms/ppo/multi_agent_cartpole_ppo.py

@@ -0,0 +1,57 @@
+from ray.rllib.algorithms.ppo import PPOConfig
+from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
+from ray.rllib.examples.envs.classes.multi_agent import MultiAgentCartPole
+from ray.rllib.examples.utils import (
+    add_rllib_example_script_args,
+    run_rllib_example_script_experiment,
+)
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+from ray.tune.registry import register_env
+
+parser = add_rllib_example_script_args()
+parser.set_defaults(
+    num_agents=2,
+)
+# Use `parser` to add your own custom command line options to this script
+# and (if needed) use their values to set up `config` below.
+args = parser.parse_args()
+
+register_env("multi_agent_cartpole", lambda cfg: MultiAgentCartPole(config=cfg))
+
+config = (
+    PPOConfig()
+    .environment("multi_agent_cartpole", env_config={"num_agents": args.num_agents})
+    .rl_module(
+        model_config=DefaultModelConfig(
+            fcnet_hiddens=[32],
+            fcnet_activation="linear",
+            vf_share_layers=True,
+        ),
+    )
+    .env_runners(
+        num_envs_per_env_runner=2,
+    )
+    .training(
+        lr=0.0003,
+        num_epochs=6,
+        vf_loss_coeff=0.01,
+    )
+    .multi_agent(
+        policy_mapping_fn=lambda aid, *arg, **kw: f"p{aid}",
+        policies={f"p{i}" for i in range(args.num_agents)},
+    )
+)
+
+stop = {
+    NUM_ENV_STEPS_SAMPLED_LIFETIME: 400000,
+    # Divide by num_agents to get actual return per agent.
+    f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 450.0 * (args.num_agents or 1),
+}
+
+
+if __name__ == "__main__":
+    run_rllib_example_script_experiment(config, args, stop=stop)

rllib/examples/algorithms/ppo/multi_agent_footsies_ppo.py

@@ -0,0 +1,264 @@
+"""
+Multi-agent RLlib Footsies Example (PPO)
+
+About:
+    - Example is based on the Footsies environment (https://github.com/chasemcd/FootsiesGym).
+    - Footsies is a two-player fighting game where each player controls a character and tries to hit the opponent while avoiding being hit.
+    - Footsies is a zero-sum game, when one player wins (+1 reward) the other loses (-1 reward).
+
+Summary:
+    - Main policy is an LSTM-based policy.
+    - Training algorithm is PPO.
+
+Training:
+    - Training is governed by adding new, more complex opponents to the mix as the main policy reaches a certain win rate threshold against the current opponent.
+    - Current opponent is always the newest opponent added to the mix.
+    - Training starts with a very simple opponent: "noop" (does nothing), then progresses to "back" (only moves backwards). These are the fixed (very simple) policies that are used to kick off the training.
+    - After "random", new opponents are frozen copies of the main policy at different training stages. They will be added to the mix as "lstm_v0", "lstm_v1", etc.
+    - In this way - after kick-starting the training with fixed simple opponents - the main policy will play against a version of itself from an earlier training stage.
+    - The main policy has to achieve the win rate threshold against the current opponent to add a new opponent to the mix.
+    - Training concludes when the target mix size is reached.
+
+Evaluation:
+    - Evaluation is performed against the current (newest) opponent.
+    - Evaluation runs for a fixed number of episodes at the end of each training iteration.
+
+"""
+import functools
+from pathlib import Path
+
+from ray.rllib.algorithms.ppo import PPOConfig
+from ray.rllib.core.rl_module import MultiRLModuleSpec, RLModuleSpec
+from ray.rllib.env.multi_agent_env_runner import MultiAgentEnvRunner
+from ray.rllib.examples.envs.classes.multi_agent.footsies.fixed_rlmodules import (
+    BackFixedRLModule,
+    NoopFixedRLModule,
+)
+from ray.rllib.examples.envs.classes.multi_agent.footsies.footsies_env import (
+    env_creator,
+)
+from ray.rllib.examples.envs.classes.multi_agent.footsies.utils import (
+    Matchmaker,
+    Matchup,
+    MetricsLoggerCallback,
+    MixManagerCallback,
+    platform_for_binary_to_download,
+)
+from ray.rllib.examples.rl_modules.classes.lstm_containing_rlm import (
+    LSTMContainingRLModule,
+)
+from ray.rllib.examples.utils import (
+    add_rllib_example_script_args,
+    run_rllib_example_script_experiment,
+)
+from ray.rllib.utils.metrics import NUM_ENV_STEPS_SAMPLED_LIFETIME
+from ray.tune.registry import register_env
+from ray.tune.result import TRAINING_ITERATION
+
+# setting two default stopping criteria:
+#    1. training_iteration (via "stop_iters")
+#    2. num_env_steps_sampled_lifetime (via "default_timesteps")
+# ...values very high to make sure that the test passes by adding
+# all required policies to the mix, not by hitting the iteration limit.
+# Our main stopping criterion is "target_mix_size" (see an argument below).
+parser = add_rllib_example_script_args(
+    default_iters=500,
+    default_timesteps=5_000_000,
+)
+
+parser.add_argument(
+    "--train-start-port",
+    type=int,
+    default=45001,
+    help="First port number for the Footsies training environment server (default: 45001). Each server gets its own port.",
+)
+parser.add_argument(
+    "--eval-start-port",
+    type=int,
+    default=55001,
+    help="First port number for the Footsies evaluation environment server (default: 55001) Each server gets its own port.",
+)
+parser.add_argument(
+    "--binary-download-dir",
+    type=Path,
+    default="/tmp/ray/binaries/footsies",
+    help="Directory to download Footsies binaries (default: /tmp/ray/binaries/footsies)",
+)
+parser.add_argument(
+    "--binary-extract-dir",
+    type=Path,
+    default="/tmp/ray/binaries/footsies",
+    help="Directory to extract Footsies binaries (default: /tmp/ray/binaries/footsies)",
+)
+parser.add_argument(
+    "--win-rate-threshold",
+    type=float,
+    default=0.8,
+    help="The main policy should have at least 'win-rate-threshold' win rate against the "
+    "other policy to advance to the next level. Moving to the next level "
+    "means adding a new policy to the mix.",
+)
+parser.add_argument(
+    "--target-mix-size",
+    type=int,
+    default=5,
+    help="Target number of policies (RLModules) in the mix to consider the test passed. "
+    "The initial mix size is 2: 'main policy' vs. 'other'. "
+    "`--target-mix-size=5` means that 3 new policies will be added to the mix. "
+    "Whether to add new policy is decided by checking the '--win-rate-threshold' condition. ",
+)
+parser.add_argument(
+    "--rollout-fragment-length",
+    type=int,
+    default=256,
+    help="The length of each rollout fragment to be collected by the EnvRunners when sampling.",
+)
+parser.add_argument(
+    "--log-unity-output",
+    action="store_true",
+    help="Whether to log Unity output (from the game engine). Default is False.",
+    default=False,
+)
+parser.add_argument(
+    "--render",
+    action="store_true",
+    default=False,
+    help="Whether to render the Footsies environment. Default is False.",
+)
+
+main_policy = "lstm"
+args = parser.parse_args()
+register_env(name="FootsiesEnv", env_creator=env_creator)
+
+# Detect platform and choose appropriate binary
+binary_to_download = platform_for_binary_to_download(args.render)
+
+config = (
+    PPOConfig()
+    .reporting(
+        min_time_s_per_iteration=30,
+    )
+    .environment(
+        env="FootsiesEnv",
+        env_config={
+            "max_t": 1000,
+            "frame_skip": 4,
+            "observation_delay": 16,
+            "train_start_port": args.train_start_port,
+            "eval_start_port": args.eval_start_port,
+            "host": "localhost",
+            "binary_download_dir": args.binary_download_dir,
+            "binary_extract_dir": args.binary_extract_dir,
+            "binary_to_download": binary_to_download,
+            "log_unity_output": args.log_unity_output,
+        },
+    )
+    .learners(
+        num_learners=1,
+        num_cpus_per_learner=1,
+        num_gpus_per_learner=0,
+        num_aggregator_actors_per_learner=0,
+    )
+    .env_runners(
+        env_runner_cls=MultiAgentEnvRunner,
+        num_env_runners=args.num_env_runners or 1,
+        num_cpus_per_env_runner=0.5,
+        num_envs_per_env_runner=1,
+        batch_mode="truncate_episodes",
+        rollout_fragment_length=args.rollout_fragment_length,
+        episodes_to_numpy=False,
+        create_env_on_local_worker=True,
+    )
+    .training(
+        train_batch_size_per_learner=args.rollout_fragment_length
+        * (args.num_env_runners or 1),
+        lr=1e-4,
+        entropy_coeff=0.01,
+        num_epochs=10,
+        minibatch_size=128,
+    )
+    .multi_agent(
+        policies={
+            main_policy,
+            "noop",
+            "back",
+        },
+        # this is a starting policy_mapping_fn
+        # It will be updated by the MixManagerCallback during training.
+        policy_mapping_fn=Matchmaker(
+            [Matchup(main_policy, "noop", 1.0)]
+        ).agent_to_module_mapping_fn,
+        # we only train the main policy, this doesn't change during training.
+        policies_to_train=[main_policy],
+    )
+    .rl_module(
+        rl_module_spec=MultiRLModuleSpec(
+            rl_module_specs={
+                main_policy: RLModuleSpec(
+                    module_class=LSTMContainingRLModule,
+                    model_config={
+                        "lstm_cell_size": 128,
+                        "dense_layers": [128, 128],
+                        "max_seq_len": 64,
+                    },
+                ),
+                # for simplicity, all fixed RLModules are added to the config at the start.
+                # However, only "noop" is used at the start of training,
+                # the others are added to the mix later by the MixManagerCallback.
+                "noop": RLModuleSpec(module_class=NoopFixedRLModule),
+                "back": RLModuleSpec(module_class=BackFixedRLModule),
+            },
+        )
+    )
+    .evaluation(
+        evaluation_num_env_runners=args.evaluation_num_env_runners or 1,
+        evaluation_sample_timeout_s=120,
+        evaluation_interval=1,
+        evaluation_duration=10,  # 10 episodes is enough to get a good win rate estimate
+        evaluation_duration_unit="episodes",
+        evaluation_parallel_to_training=False,
+        # we may add new RLModules to the mix at the end of the evaluation stage.
+        # Running evaluation in parallel may result in training for one more iteration on the old mix.
+        evaluation_force_reset_envs_before_iteration=True,
+        evaluation_config={
+            "env_config": {"env-for-evaluation": True},
+        },  # evaluation_config is used to add an argument to the env creator.
+    )
+    .callbacks(
+        [
+            functools.partial(
+                MetricsLoggerCallback,
+                main_policy=main_policy,
+            ),
+            functools.partial(
+                MixManagerCallback,
+                win_rate_threshold=args.win_rate_threshold,
+                main_policy=main_policy,
+                target_mix_size=args.target_mix_size,
+                starting_modules=[main_policy, "noop"],
+                fixed_modules_progression_sequence=(
+                    "noop",
+                    "back",
+                ),
+            ),
+        ]
+    )
+)
+
+# stopping criteria to be passed to Ray Tune. The main stopping criterion is "mix_size".
+# "mix_size" is reported at the end of each training iteration by the MixManagerCallback.
+stop = {
+    NUM_ENV_STEPS_SAMPLED_LIFETIME: args.stop_timesteps,
+    TRAINING_ITERATION: args.stop_iters,
+    "mix_size": args.target_mix_size,
+}
+
+if __name__ == "__main__":
+    results = run_rllib_example_script_experiment(
+        base_config=config,
+        args=args,
+        stop=stop,
+        success_metric={
+            "mix_size": args.target_mix_size
+        },  # pass the success metric for RLlib's testing framework
+    )

rllib/examples/algorithms/ppo/multi_agent_pendulum_ppo.py

@@ -0,0 +1,58 @@
+from ray.rllib.algorithms.ppo import PPOConfig
+from ray.rllib.connectors.env_to_module import MeanStdFilter
+from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
+from ray.rllib.examples.envs.classes.multi_agent import MultiAgentPendulum
+from ray.rllib.examples.utils import (
+    add_rllib_example_script_args,
+    run_rllib_example_script_experiment,
+)
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+from ray.tune.registry import register_env
+
+parser = add_rllib_example_script_args(default_timesteps=500000)
+parser.set_defaults(
+    num_agents=2,
+)
+# Use `parser` to add your own custom command line options to this script
+# and (if needed) use their values to set up `config` below.
+args = parser.parse_args()
+
+register_env("multi_agent_pendulum", lambda cfg: MultiAgentPendulum(config=cfg))
+
+config = (
+    PPOConfig()
+    .environment("multi_agent_pendulum", env_config={"num_agents": args.num_agents})
+    .env_runners(
+        env_to_module_connector=lambda env, spaces, device: MeanStdFilter(
+            multi_agent=True
+        ),
+    )
+    .training(
+        train_batch_size_per_learner=1024,
+        minibatch_size=128,
+        lr=0.0002 * (args.num_learners or 1) ** 0.5,
+        gamma=0.95,
+        lambda_=0.5,
+    )
+    .rl_module(
+        model_config=DefaultModelConfig(fcnet_activation="relu"),
+    )
+    .multi_agent(
+        policy_mapping_fn=lambda aid, *arg, **kw: f"p{aid}",
+        policies={f"p{i}" for i in range(args.num_agents)},
+    )
+)
+
+stop = {
+    NUM_ENV_STEPS_SAMPLED_LIFETIME: args.stop_timesteps,
+    # Divide by num_agents to get actual return per agent.
+    f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": -300.0 * (args.num_agents or 1),
+}
+
+
+if __name__ == "__main__":
+    run_rllib_example_script_experiment(config, args, stop=stop)

rllib/examples/algorithms/ppo/multi_agent_stateless_cartpole_ppo.py

@@ -0,0 +1,63 @@
+from ray.rllib.algorithms.ppo import PPOConfig
+from ray.rllib.connectors.env_to_module import MeanStdFilter
+from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
+from ray.rllib.examples.envs.classes.multi_agent import MultiAgentStatelessCartPole
+from ray.rllib.examples.utils import (
+    add_rllib_example_script_args,
+    run_rllib_example_script_experiment,
+)
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+from ray.tune import register_env
+
+parser = add_rllib_example_script_args(default_timesteps=4000000)
+parser.set_defaults(
+    num_agents=2,
+    num_env_runners=3,
+)
+# Use `parser` to add your own custom command line options to this script
+# and (if needed) use their values to set up `config` below.
+args = parser.parse_args()
+
+register_env(
+    "multi_stateless_cart",
+    lambda _: MultiAgentStatelessCartPole({"num_agents": args.num_agents}),
+)
+
+config = (
+    PPOConfig()
+    .environment("multi_stateless_cart")
+    .env_runners(
+        env_to_module_connector=lambda env, spaces, device: MeanStdFilter(
+            multi_agent=True
+        ),
+    )
+    .training(
+        lr=0.0003 * ((args.num_learners or 1) ** 0.5),
+        num_epochs=6,
+        vf_loss_coeff=0.05,
+    )
+    .rl_module(
+        model_config=DefaultModelConfig(
+            use_lstm=True,
+            max_seq_len=20,
+        ),
+    )
+    .multi_agent(
+        policy_mapping_fn=lambda aid, *arg, **kw: f"p{aid}",
+        policies={f"p{i}" for i in range(args.num_agents)},
+    )
+)
+
+stop = {
+    NUM_ENV_STEPS_SAMPLED_LIFETIME: args.stop_timesteps,
+    # Divide by num_agents to get actual return per agent.
+    f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 300.0 * (args.num_agents or 1),
+}
+
+
+if __name__ == "__main__":
+    run_rllib_example_script_experiment(config, args, stop=stop)

rllib/examples/algorithms/ppo/pendulum_ppo.py

@@ -0,0 +1,37 @@
+from ray.rllib.algorithms.ppo import PPOConfig
+from ray.rllib.connectors.env_to_module import MeanStdFilter
+from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
+from ray.rllib.examples.utils import (
+    add_rllib_example_script_args,
+    run_rllib_example_script_experiment,
+)
+
+parser = add_rllib_example_script_args(default_timesteps=400000, default_reward=-300)
+# Use `parser` to add your own custom command line options to this script
+# and (if needed) use their values to set up `config` below.
+args = parser.parse_args()
+
+config = (
+    PPOConfig()
+    .environment("Pendulum-v1")
+    .env_runners(
+        num_env_runners=2,
+        num_envs_per_env_runner=20,
+        env_to_module_connector=lambda env, spaces, device: MeanStdFilter(),
+    )
+    .training(
+        train_batch_size_per_learner=1024,
+        minibatch_size=128,
+        lr=0.0002 * (args.num_learners or 1) ** 0.5,
+        gamma=0.95,
+        lambda_=0.5,
+        # num_epochs=8,
+    )
+    .rl_module(
+        model_config=DefaultModelConfig(fcnet_activation="relu"),
+    )
+)
+
+
+if __name__ == "__main__":
+    run_rllib_example_script_experiment(config, args)

rllib/examples/algorithms/ppo/stateless_cartpole_ppo.py

@@ -0,0 +1,43 @@
+from ray.rllib.algorithms.ppo import PPOConfig
+from ray.rllib.connectors.env_to_module import MeanStdFilter
+from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
+from ray.rllib.examples.envs.classes.stateless_cartpole import StatelessCartPole
+from ray.rllib.examples.utils import (
+    add_rllib_example_script_args,
+    run_rllib_example_script_experiment,
+)
+
+parser = add_rllib_example_script_args(
+    default_timesteps=2000000,
+    default_reward=350.0,
+)
+parser.set_defaults(
+    num_env_runners=3,
+)
+# Use `parser` to add your own custom command line options to this script
+# and (if needed) use their values to set up `config` below.
+args = parser.parse_args()
+
+config = (
+    PPOConfig()
+    .environment(StatelessCartPole)
+    .env_runners(
+        env_to_module_connector=lambda env, spaces, device: MeanStdFilter(),
+    )
+    .training(
+        lr=0.0003 * ((args.num_learners or 1) ** 0.5),
+        num_epochs=6,
+        vf_loss_coeff=0.05,
+    )
+    .rl_module(
+        model_config=DefaultModelConfig(
+            vf_share_layers=False,
+            use_lstm=True,
+            max_seq_len=20,
+        ),
+    )
+)
+
+
+if __name__ == "__main__":
+    run_rllib_example_script_experiment(config, args)