chore: import upstream snapshot with attribution

rllib/examples/_old_api_stack/algorithms/atari-dist-dqn.yaml

@@ -0,0 +1,35 @@
+# @OldAPIStack
+atari-dist-dqn:
+    env:
+        grid_search:
+            - ale_py:ALE/Breakout-v5
+            - ale_py:ALE/BeamRider-v5
+            - ale_py:ALE/Qbert-v5
+            - ale_py:ALE/SpaceInvaders-v5
+    run: DQN
+    config:
+        # Make analogous to old v4 + NoFrameskip.
+        env_config:
+            frameskip: 1
+            full_action_space: false
+            repeat_action_probability: 0.0
+        double_q: false
+        dueling: false
+        num_atoms: 51
+        noisy: false
+        replay_buffer_config:
+          type: MultiAgentReplayBuffer
+          capacity: 1000000
+        num_steps_sampled_before_learning_starts: 20000
+        n_step: 1
+        target_network_update_freq: 8000
+        lr: .0000625
+        adam_epsilon: .00015
+        hiddens: [512]
+        rollout_fragment_length: 4
+        train_batch_size: 32
+        exploration_config:
+          epsilon_timesteps: 200000
+          final_epsilon: 0.01
+        num_gpus: 0.2
+        min_sample_timesteps_per_iteration: 10000

rllib/examples/_old_api_stack/algorithms/atari-dqn.yaml

@@ -0,0 +1,39 @@
+# @OldAPIStack
+# Runs on a single g3.4xl node
+# See https://github.com/ray-project/rl-experiments for results
+atari-basic-dqn:
+    env:
+        grid_search:
+            - ale_py:ALE/Breakout-v5
+            - ale_py:ALE/BeamRider-v5
+            - ale_py:ALE/Qbert-v5
+            - ale_py:ALE/SpaceInvaders-v5
+    run: DQN
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        # Make analogous to old v4 + NoFrameskip.
+        env_config:
+            frameskip: 1
+            full_action_space: false
+            repeat_action_probability: 0.0
+        double_q: false
+        dueling: false
+        num_atoms: 1
+        noisy: false
+        replay_buffer_config:
+          type: MultiAgentReplayBuffer
+          capacity: 1000000
+        num_steps_sampled_before_learning_starts: 20000
+        n_step: 1
+        target_network_update_freq: 8000
+        lr: .0000625
+        adam_epsilon: .00015
+        hiddens: [512]
+        rollout_fragment_length: 4
+        train_batch_size: 32
+        exploration_config:
+          epsilon_timesteps: 200000
+          final_epsilon: 0.01
+        num_gpus: 0.2
+        min_sample_timesteps_per_iteration: 10000

rllib/examples/_old_api_stack/algorithms/atari-duel-ddqn.yaml

@@ -0,0 +1,39 @@
+# @OldAPIStack
+# Runs on a single g3.4xl node
+# See https://github.com/ray-project/rl-experiments for results
+dueling-ddqn:
+    env:
+        grid_search:
+            - ale_py:ALE/Breakout-v5
+            - ale_py:ALE/BeamRider-v5
+            - ale_py:ALE/Qbert-v5
+            - ale_py:ALE/SpaceInvaders-v5
+    run: DQN
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        # Make analogous to old v4 + NoFrameskip.
+        env_config:
+            frameskip: 1
+            full_action_space: false
+            repeat_action_probability: 0.0
+        double_q: true
+        dueling: true
+        num_atoms: 1
+        noisy: false
+        replay_buffer_config:
+          type: MultiAgentReplayBuffer
+          capacity: 1000000
+        num_steps_sampled_before_learning_starts: 20000
+        n_step: 1
+        target_network_update_freq: 8000
+        lr: .0000625
+        adam_epsilon: .00015
+        hiddens: [512]
+        rollout_fragment_length: 4
+        train_batch_size: 32
+        exploration_config:
+          epsilon_timesteps: 200000
+          final_epsilon: 0.01
+        num_gpus: 0.2
+        min_sample_timesteps_per_iteration: 10000

rllib/examples/_old_api_stack/algorithms/atari-impala-large.yaml

@@ -0,0 +1,28 @@
+# @OldAPIStack
+# Runs on a g3.16xl node with 5 m5.24xl workers
+# Takes roughly 10 minutes.
+atari-impala:
+    env:
+        grid_search:
+            - ale_py:ALE/Breakout-v5
+            - ale_py:ALE/BeamRider-v5
+            - ale_py:ALE/Qbert-v5
+            - ale_py:ALE/SpaceInvaders-v5
+    run: IMPALA
+    stop:
+        timesteps_total: 3000000
+    config:
+        # Make analogous to old v4 + NoFrameskip.
+        env_config:
+            frameskip: 1
+            full_action_space: false
+            repeat_action_probability: 0.0
+        rollout_fragment_length: 50
+        train_batch_size: 500
+        num_env_runners: 128
+        num_envs_per_env_runner: 5
+        clip_rewards: True
+        lr_schedule: [
+            [0, 0.0005],
+            [20000000, 0.000000000001],
+        ]

rllib/examples/_old_api_stack/algorithms/atari-impala-multi-gpu.yaml

@@ -0,0 +1,25 @@
+# @OldAPIStack
+# Runs on a p2.8xlarge single head node machine.
+# Should reach ~400 reward in about 1h and after 15-20M ts.
+atari-impala:
+    env: ale_py:ALE/Breakout-v5
+    run: IMPALA
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        # Make analogous to old v4 + NoFrameskip.
+        env_config:
+            frameskip: 1
+            full_action_space: false
+            repeat_action_probability: 0.0
+        rollout_fragment_length: 50
+        train_batch_size: 4000
+        num_gpus: 4
+        num_env_runners: 31
+        num_gpus_per_env_runner: 0  # works also for partial GPUs (<1.0) per worker
+        num_envs_per_env_runner: 5
+        clip_rewards: True
+        lr_schedule: [
+            [0, 0.0005],
+            [20000000, 0.000000000001],
+        ]

rllib/examples/_old_api_stack/algorithms/atari-impala.yaml

@@ -0,0 +1,26 @@
+# @OldAPIStack
+# Runs on a g3.16xl node with 3 m4.16xl workers
+# See https://github.com/ray-project/rl-experiments for results
+atari-impala:
+    env:
+        grid_search:
+            - ale_py:ALE/Breakout-v5
+            - ale_py:ALE/BeamRider-v5
+            - ale_py:ALE/Qbert-v5
+            - ale_py:ALE/SpaceInvaders-v5
+    run: IMPALA
+    config:
+        # Make analogous to old v4 + NoFrameskip.
+        env_config:
+            frameskip: 1
+            full_action_space: false
+            repeat_action_probability: 0.0
+        rollout_fragment_length: 50
+        train_batch_size: 500
+        num_env_runners: 32
+        num_envs_per_env_runner: 5
+        clip_rewards: True
+        lr_schedule: [
+            [0, 0.0005],
+            [20000000, 0.000000000001],
+        ]

rllib/examples/_old_api_stack/algorithms/atari-sac.yaml

@@ -0,0 +1,55 @@
+# @OldAPIStack
+# Run e.g. on a g3.16xlarge (4 GPUs) with `num_gpus=1` (1 for each trial;
+# MsPacman torch + tf; Pong torch + tf).
+# Uses the hyperparameters published in [2] (see rllib/algorithms/sac/README.md).
+atari-sac-tf-and-torch:
+    env:
+        grid_search:
+            - ale_py:ALE/MsPacman-v5
+            - ale_py:ALE/Pong-v5
+    run: SAC
+    stop:
+        timesteps_total: 20000000
+    config:
+        # Works for both torch and tf.
+        framework:
+            grid_search: [tf, torch]
+        env_config:
+            frameskip: 1  # no frameskip
+        gamma: 0.99
+        q_model_config:
+            hidden_activation: relu
+            hidden_layer_sizes: [512]
+        policy_model_config:
+            hidden_activation: relu
+            hidden_layer_sizes: [512]
+        # Do hard syncs.
+        # Soft-syncs seem to work less reliably for discrete action spaces.
+        tau: 1.0
+        target_network_update_freq: 8000
+        # auto = 0.98 * -log(1/|A|)
+        target_entropy: auto
+        clip_rewards: 1.0
+        n_step: 1
+        rollout_fragment_length: 1
+        replay_buffer_config:
+          type: MultiAgentPrioritizedReplayBuffer
+          capacity: 1000000
+          # How many steps of the model to sample before learning starts.
+          # If True prioritized replay buffer will be used.
+          prioritized_replay_alpha: 0.6
+          prioritized_replay_beta: 0.4
+          prioritized_replay_eps: 1e-6
+        num_steps_sampled_before_learning_starts: 100000
+        train_batch_size: 64
+        min_sample_timesteps_per_iteration: 4
+        # Paper uses 20k random timesteps, which is not exactly the same, but
+        # seems to work nevertheless. We use 100k here for the longer Atari
+        # runs (DQN style: filling up the buffer a bit before learning).
+        optimization:
+            actor_learning_rate: 0.0003
+            critic_learning_rate: 0.0003
+            entropy_learning_rate: 0.0003
+        num_env_runners: 0
+        num_gpus: 1
+        metrics_num_episodes_for_smoothing: 5

rllib/examples/_old_api_stack/algorithms/cartpole-appo-separate-losses.py

@@ -0,0 +1,44 @@
+# @OldAPIStack
+from ray import tune
+from ray.rllib.algorithms.appo import APPOConfig
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+
+stop = {
+    f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 400,
+    f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 200000,
+}
+
+config = (
+    APPOConfig()
+    .api_stack(
+        enable_rl_module_and_learner=False,
+        enable_env_runner_and_connector_v2=False,
+    )
+    .environment("CartPole-v1")
+    # Switch on >1 loss/optimizer API for TFPolicy and EagerTFPolicy.
+    .experimental(_tf_policy_handles_more_than_one_loss=True)
+    .training(
+        # APPO will produce two separate loss terms: policy loss + value function loss.
+        _separate_vf_optimizer=True,
+        # Separate learning rate (and schedule) for the value function branch.
+        _lr_vf=tune.grid_search([0.00075, [[0, 0.00075], [100000, 0.0003]]]),
+        num_epochs=6,
+        # `vf_loss_coeff` will be ignored anyways as we use separate loss terms.
+        vf_loss_coeff=0.01,
+        vtrace=True,
+        model={
+            # Make sure we really have completely separate branches.
+            "vf_share_layers": False,
+        },
+    )
+    .env_runners(
+        num_envs_per_env_runner=5,
+        num_env_runners=1,
+        observation_filter="MeanStdFilter",
+    )
+    .resources(num_gpus=0)
+)

rllib/examples/_old_api_stack/algorithms/cartpole-bc.yaml

@@ -0,0 +1,26 @@
+# @OldAPIStack
+# To generate training data, first run:
+# $ ./train.py --run=PPO --env=CartPole-v1 \
+#      --stop='{"timesteps_total": 50000}' \
+#      --config='{"output": "dataset", "output_config": {"format": "json", "path": "/tmp/out", "max_num_samples_per_file": 1}, "batch_mode": "complete_episodes"}'
+cartpole-bc:
+    env: CartPole-v1
+    run: BC
+    stop:
+        timesteps_total: 500000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        enable_rl_module_and_learner: false
+        enable_env_runner_and_connector_v2: false
+        # In order to evaluate on an actual environment, use these following
+        # settings:
+        evaluation_num_env_runners: 1
+        evaluation_interval: 1
+        evaluation_config:
+            input: sampler
+        # The historic (offline) data file from the PPO run (at the top).
+        input: dataset
+        input_config:
+            format: json
+            paths: /tmp/out

rllib/examples/_old_api_stack/algorithms/cartpole-crashing-and-stalling-recreate-workers-appo.py

@@ -0,0 +1,82 @@
+# @OldAPIStack
+"""
+Tests, whether APPO can learn in a fault-tolerant fashion.
+
+Workers will be configured to automatically get recreated upon failures (here: within
+the environment).
+The environment we use here is configured to crash with a certain probability on each
+`step()` and/or `reset()` call. Additionally, the environment is configured to stall
+with a configured probability on each `step()` call for a certain amount of time.
+"""
+from gymnasium.wrappers import TimeLimit
+
+from ray import tune
+from ray.rllib.algorithms.appo import APPOConfig
+from ray.rllib.examples.envs.classes.cartpole_crashing import CartPoleCrashing
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    EVALUATION_RESULTS,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+
+tune.register_env(
+    "env",
+    lambda cfg: TimeLimit(CartPoleCrashing(cfg), max_episode_steps=500),
+)
+
+config = (
+    APPOConfig()
+    .api_stack(
+        enable_rl_module_and_learner=False,
+        enable_env_runner_and_connector_v2=False,
+    )
+    .environment(
+        "env",
+        env_config={
+            "p_crash": 0.0001,  # prob to crash during step()
+            "p_crash_reset": 0.001,  # prob to crash during reset()
+            "crash_on_worker_indices": [1, 2],
+            "init_time_s": 2.0,
+            "p_stall": 0.0005,  # prob to stall during step()
+            "p_stall_reset": 0.001,  # prob to stall during reset()
+            "stall_time_sec": (2, 5),  # stall between 2 and 10sec.
+            "stall_on_worker_indices": [2, 3],
+        },
+    )
+    .env_runners(
+        num_env_runners=1,
+        num_envs_per_env_runner=1,
+    )
+    # Switch on resiliency (recreate any failed worker).
+    .fault_tolerance(
+        restart_failed_env_runners=True,
+    )
+    .evaluation(
+        evaluation_num_env_runners=4,
+        evaluation_interval=1,
+        evaluation_duration=25,
+        evaluation_duration_unit="episodes",
+        evaluation_parallel_to_training=True,
+        evaluation_config=APPOConfig.overrides(
+            explore=False,
+            env_config={
+                # Make eval workers solid.
+                # This test is to prove that we can learn with crashing envs,
+                # not evaluate with crashing envs.
+                "p_crash": 0.0,
+                "p_crash_reset": 0.0,
+                "init_time_s": 0.0,
+                "p_stall": 0.01,
+                "stall_time_sec": 300,  # stall for 5min.
+                "p_stall_reset": 0.0,
+                "stall_on_worker_indices": [1, 2],
+            },
+        ),
+    )
+)
+
+stop = {
+    f"{EVALUATION_RESULTS}/{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 500.0,
+    f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 2000000,
+}

rllib/examples/_old_api_stack/algorithms/cartpole-crashing-recreate-workers-appo.py

@@ -0,0 +1,69 @@
+# @OldAPIStack
+"""
+Tests, whether APPO can learn in a fault-tolerant fashion.
+
+Workers will be configured to automatically get recreated upon failures (here: within
+the environment).
+The environment we use here is configured to crash with a certain probability on each
+`step()` and/or `reset()` call.
+"""
+from ray import tune
+from ray.rllib.algorithms.appo import APPOConfig
+from ray.rllib.examples.envs.classes.cartpole_crashing import CartPoleCrashing
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    EVALUATION_RESULTS,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+
+tune.register_env("env", lambda cfg: CartPoleCrashing(cfg))
+
+
+stop = {
+    f"{EVALUATION_RESULTS}/{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 400.0,
+    f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 250000,
+}
+
+config = (
+    APPOConfig()
+    .api_stack(
+        enable_rl_module_and_learner=False,
+        enable_env_runner_and_connector_v2=False,
+    )
+    .environment(
+        "env",
+        env_config={
+            # Crash roughly every 500 ts.
+            "p_crash": 0.0005,  # prob to crash during step()
+            "p_crash_reset": 0.005,  # prob to crash during reset()
+            "crash_on_worker_indices": [1, 2],
+        },
+    )
+    .env_runners(
+        num_env_runners=3,
+        num_envs_per_env_runner=1,
+    )
+    # Switch on resiliency (recreate any failed worker).
+    .fault_tolerance(
+        restart_failed_env_runners=True,
+    )
+    .evaluation(
+        evaluation_num_env_runners=1,
+        evaluation_interval=1,
+        evaluation_duration=25,
+        evaluation_duration_unit="episodes",
+        evaluation_parallel_to_training=True,
+        evaluation_config=APPOConfig.overrides(
+            explore=False,
+            env_config={
+                # Make eval workers solid.
+                # This test is to prove that we can learn with crashing envs,
+                # not evaluate with crashing envs.
+                "p_crash": 0.0,
+                "p_crash_reset": 0.0,
+                "init_time_s": 0.0,
+            },
+        ),
+    )
+)

rllib/examples/_old_api_stack/algorithms/cartpole-dqn-fake-gpus.yaml

@@ -0,0 +1,20 @@
+# @OldAPIStack
+cartpole-dqn-fake-gpus:
+    env: CartPole-v1
+    run: DQN
+    stop:
+        env_runners/episode_return_mean: 150
+        training_iteration: 400
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        model:
+            fcnet_hiddens: [64]
+            fcnet_activation: linear
+        n_step: 3
+        # Double batch size (2 GPUs).
+        train_batch_size: 64
+
+        # Fake 2 GPUs.
+        num_gpus: 2
+        _fake_gpus: true

rllib/examples/_old_api_stack/algorithms/cartpole-dqn-param-noise.yaml

@@ -0,0 +1,20 @@
+# @OldAPIStack
+cartpole-dqn-w-param-noise:
+    env: CartPole-v1
+    run: DQN
+    stop:
+        env_runners/episode_return_mean: 150
+        timesteps_total: 300000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        exploration_config:
+            type: ParameterNoise
+            random_timesteps: 10000
+            initial_stddev: 1.0
+        batch_mode: complete_episodes
+        lr: 0.0008
+        num_env_runners: 0
+        model:
+            fcnet_hiddens: [32, 32]
+            fcnet_activation: tanh

rllib/examples/_old_api_stack/algorithms/cartpole-dqn-softq.yaml

@@ -0,0 +1,17 @@
+# @OldAPIStack
+cartpole-dqn:
+    env: CartPole-v1
+    run: DQN
+    stop:
+        env_runners/episode_return_mean: 150
+        timesteps_total: 100000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        model:
+            fcnet_hiddens: [64]
+            fcnet_activation: linear
+        n_step: 3
+        exploration_config:
+            type: SoftQ
+            temperature: 0.5

rllib/examples/_old_api_stack/algorithms/cartpole-dqn.yaml

@@ -0,0 +1,14 @@
+# @OldAPIStack
+cartpole-dqn:
+    env: CartPole-v1
+    run: DQN
+    stop:
+        env_runners/episode_return_mean: 100
+        timesteps_total: 100000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        model:
+            fcnet_hiddens: [64]
+            fcnet_activation: linear
+        n_step: 3

rllib/examples/_old_api_stack/algorithms/cartpole-impala-separate-losses.py

@@ -0,0 +1,44 @@
+# @OldAPIStack
+from ray.rllib.algorithms.impala import IMPALAConfig
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+
+stop = {
+    f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 150,
+    f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 200000,
+}
+
+config = (
+    IMPALAConfig()
+    .api_stack(
+        enable_rl_module_and_learner=False,
+        enable_env_runner_and_connector_v2=False,
+    )
+    .environment("CartPole-v1")
+    # Switch on >1 loss/optimizer API for TFPolicy and EagerTFPolicy.
+    .experimental(_tf_policy_handles_more_than_one_loss=True)
+    .training(
+        # IMPALA will produce two separate loss terms: policy loss + value function
+        # loss.
+        _separate_vf_optimizer=True,
+        # Separate learning rate for the value function branch.
+        _lr_vf=0.00075,
+        num_epochs=6,
+        # `vf_loss_coeff` will be ignored anyways as we use separate loss terms.
+        vf_loss_coeff=0.01,
+        vtrace=True,
+        model={
+            # Make sure we really have completely separate branches.
+            "vf_share_layers": False,
+        },
+    )
+    .env_runners(
+        num_envs_per_env_runner=5,
+        num_env_runners=1,
+        observation_filter="MeanStdFilter",
+    )
+    .resources(num_gpus=0)
+)

rllib/examples/_old_api_stack/algorithms/cartpole-marwil.yaml

@@ -0,0 +1,22 @@
+# @OldAPIStack
+# To generate training data, first run:
+# $ ./train.py --run=PPO --env=CartPole-v1 \
+#      --stop='{"timesteps_total": 50000}' \
+#      --config='{"output": "/tmp/out", "batch_mode": "complete_episodes"}'
+cartpole-marwil:
+    env: CartPole-v1
+    run: MARWIL
+    stop:
+        timesteps_total: 500000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        # In order to evaluate on an actual environment, use these following
+        # settings:
+        evaluation_num_env_runners: 1
+        evaluation_interval: 1
+        evaluation_config:
+            input: sampler
+        beta: 1.0  # Compare to behavior cloning (beta=0.0).
+        # The historic (offline) data file from the PPO run (at the top).
+        input: /tmp/out

rllib/examples/_old_api_stack/algorithms/cartpole-sac.yaml

@@ -0,0 +1,22 @@
+# @OldAPIStack
+cartpole-sac:
+    env: CartPole-v1
+    run: SAC
+    stop:
+        env_runners/episode_return_mean: 150.0
+        timesteps_total: 100000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        gamma: 0.95
+        target_network_update_freq: 32
+        tau: 1.0
+        # initial_alpha: 0.5
+        train_batch_size: 32
+        optimization:
+            actor_learning_rate: 0.005
+            critic_learning_rate: 0.005
+            entropy_learning_rate: 0.0001
+        # grad_norm_clipping: 40.0
+        # evaluation_config:
+        #     explore: true

rllib/examples/_old_api_stack/algorithms/frozenlake-appo-vtrace.yaml

@@ -0,0 +1,33 @@
+# @OldAPIStack
+frozenlake-appo-vtrace:
+    env: FrozenLake-v1
+    run: APPO
+    stop:
+        env_runners/episode_return_mean: 0.99
+        timesteps_total: 1000000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+
+        # Sparse reward environment (short horizon).
+        env_config:
+            desc:
+                - SFFFFFFF
+                - FFFFFFFF
+                - FFFFFFFF
+                - FFFFFFFF
+                - FFFFFFFF
+                - FFFFFFFF
+                - FFFFFFFF
+                - FFFFFFFG
+            is_slippery: false
+        horizon: 20
+        rollout_fragment_length: 10
+        batch_mode: complete_episodes
+        vtrace: true
+
+        num_envs_per_env_runner: 5
+        num_env_runners: 4
+        num_gpus: 0
+        num_epochs: 1
+        vf_loss_coeff: 0.01

rllib/examples/_old_api_stack/algorithms/halfcheetah-bc.yaml

@@ -0,0 +1,50 @@
+# @OldAPIStack
+halfcheetah_bc:
+    env:
+        grid_search:
+            #- ray.rllib.examples.envs.classes.d4rl_env.halfcheetah_random
+            #- ray.rllib.examples.envs.classes.d4rl_env.halfcheetah_medium
+            - ray.rllib.examples.envs.classes.d4rl_env.halfcheetah_expert
+            #- ray.rllib.examples.envs.classes.d4rl_env.halfcheetah_medium_replay
+    run: CQL
+    config:
+        # SAC Configs
+        #input: d4rl.halfcheetah-random-v0
+        #input: d4rl.halfcheetah-medium-v0
+        input: d4rl.halfcheetah-expert-v0
+        #input: d4rl.halfcheetah-medium-replay-v0
+        framework: torch
+        q_model_config:
+          fcnet_activation: relu
+          fcnet_hiddens: [256, 256, 256]
+        policy_model_config:
+          fcnet_activation: relu
+          fcnet_hiddens: [256, 256, 256]
+        tau: 0.005
+        target_entropy: auto
+        n_step: 1
+        rollout_fragment_length: 1
+        replay_buffer_config:
+          type: MultiAgentReplayBuffer
+        num_steps_sampled_before_learning_starts: 10
+        train_batch_size: 256
+        target_network_update_freq: 0
+        min_train_timesteps_per_iteration: 1000
+        optimization:
+          actor_learning_rate: 0.0001
+          critic_learning_rate: 0.0003
+          entropy_learning_rate: 0.0001
+        num_env_runners: 0
+        num_gpus: 1
+        clip_actions: false
+        normalize_actions: true
+        evaluation_interval: 1
+        metrics_num_episodes_for_smoothing: 5
+        # CQL Configs
+        min_q_weight: 5.0
+        bc_iters: 200000000
+        temperature: 1.0
+        num_actions: 10
+        lagrangian: False
+        evaluation_config:
+            input: sampler

rllib/examples/_old_api_stack/algorithms/halfcheetah-cql.yaml

@@ -0,0 +1,51 @@
+# @OldAPIStack
+halfcheetah_cql:
+    env:
+        grid_search:
+            #- ray.rllib.examples.envs.classes.d4rl_env.halfcheetah_random
+            #- ray.rllib.examples.envs.classes.d4rl_env.halfcheetah_medium
+            - ray.rllib.examples.envs.classes.d4rl_env.halfcheetah_expert
+            #- ray.rllib.examples.envs.classes.d4rl_env.halfcheetah_medium_replay
+    run: CQL
+    config:
+        # SAC Configs
+        #input: d4rl.halfcheetah-random-v0
+        #input: d4rl.halfcheetah-medium-v0
+        input: d4rl.halfcheetah-expert-v0
+        #input: d4rl.halfcheetah-medium-replay-v0
+
+        # Works for both torch and tf.
+        framework: torch
+        q_model_config:
+            fcnet_activation: relu
+            fcnet_hiddens: [256, 256, 256]
+        policy_model_config:
+            fcnet_activation: relu
+            fcnet_hiddens: [256, 256, 256]
+        tau: 0.005
+        target_entropy: auto
+        n_step: 3
+        rollout_fragment_length: 1
+        replay_buffer_config:
+          type: MultiAgentReplayBuffer
+        num_steps_sampled_before_learning_starts: 256
+        train_batch_size: 256
+        target_network_update_freq: 0
+        min_train_timesteps_per_iteration: 1000
+        optimization:
+            actor_learning_rate: 0.0001
+            critic_learning_rate: 0.0003
+            entropy_learning_rate: 0.0001
+        num_env_runners: 0
+        num_gpus: 1
+        metrics_num_episodes_for_smoothing: 5
+        # CQL Configs
+        min_q_weight: 5.0
+        bc_iters: 20000
+        temperature: 1.0
+        num_actions: 10
+        lagrangian: False
+
+        evaluation_interval: 3
+        evaluation_config:
+            input: sampler

rllib/examples/_old_api_stack/algorithms/halfcheetah-ppo.yaml

@@ -0,0 +1,26 @@
+# @OldAPIStack
+halfcheetah-ppo:
+    env: HalfCheetah-v2
+    run: PPO
+    stop:
+        env_runners/episode_return_mean: 9800
+        time_total_s: 10800
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        gamma: 0.99
+        lambda: 0.95
+        kl_coeff: 1.0
+        num_epochs: 32
+        lr: .0003
+        vf_loss_coeff: 0.5
+        clip_param: 0.2
+        minibatch_size: 4096
+        train_batch_size: 65536
+        num_env_runners: 16
+        num_gpus: 1
+        grad_clip: 0.5
+        num_envs_per_env_runner:
+            grid_search: [16, 32]
+        batch_mode: truncate_episodes
+        observation_filter: MeanStdFilter

rllib/examples/_old_api_stack/algorithms/hopper-bc.yaml

@@ -0,0 +1,50 @@
+# @OldAPIStack
+hopper_bc:
+    env:
+        grid_search:
+            - ray.rllib.examples.envs.classes.d4rl_env.hopper_random
+            #- ray.rllib.examples.envs.classes..d4rl_env.hopper_medium
+            #- ray.rllib.examples.envs.classes..d4rl_env.hopper_expert
+            #- ray.rllib.examples.envs.classes..d4rl_env.hopper_medium_replay
+    run: CQL
+    config:
+        # SAC Configs
+        input: d4rl.hopper-random-v0
+        #input: d4rl.hopper-medium-v0
+        #input: d4rl.hopper-expert-v0
+        #input: d4rl.hopper-medium-replay-v0
+        framework: torch
+        q_model_config:
+          fcnet_activation: relu
+          fcnet_hiddens: [256, 256, 256]
+        policy_model_config:
+          fcnet_activation: relu
+          fcnet_hiddens: [256, 256, 256]
+        tau: 0.005
+        target_entropy: auto
+        n_step: 1
+        rollout_fragment_length: 1
+        replay_buffer_config:
+          type: MultiAgentReplayBuffer
+        num_steps_sampled_before_learning_starts: 10
+        train_batch_size: 256
+        target_network_update_freq: 0
+        min_train_timesteps_per_iteration: 1000
+        optimization:
+          actor_learning_rate: 0.0001
+          critic_learning_rate: 0.0003
+          entropy_learning_rate: 0.0001
+        num_env_runners: 0
+        num_gpus: 1
+        clip_actions: false
+        normalize_actions: true
+        evaluation_interval: 1
+        metrics_num_episodes_for_smoothing: 5
+        # CQL Configs
+        min_q_weight: 5.0
+        bc_iters: 200000000
+        temperature: 1.0
+        num_actions: 10
+        lagrangian: False
+        evaluation_config:
+            input: sampler

rllib/examples/_old_api_stack/algorithms/hopper-cql.yaml

@@ -0,0 +1,50 @@
+# @OldAPIStack
+hopper_cql:
+    env:
+        grid_search:
+            #- ray.rllib.examples.envs.classes.d4rl_env.hopper_random
+            - ray.rllib.examples.envs.classes.d4rl_env.hopper_medium
+            #- ray.rllib.examples.envs.classes.d4rl_env.hopper_expert
+            #- ray.rllib.examples.envs.classes.d4rl_env.hopper_medium_replay
+    run: CQL
+    config:
+        # SAC Configs
+        #input: d4rl.hopper-random-v0
+        input: d4rl.hopper-medium-v0
+        #input: d4rl.hopper-expert-v0
+        #input: d4rl.hopper-medium-replay-v0
+        framework: torch
+        q_model_config:
+          fcnet_activation: relu
+          fcnet_hiddens: [256, 256, 256]
+        policy_model_config:
+          fcnet_activation: relu
+          fcnet_hiddens: [256, 256, 256]
+        tau: 0.005
+        target_entropy: auto
+        n_step: 1
+        rollout_fragment_length: 1
+        replay_buffer_config:
+          type: MultiAgentReplayBuffer
+        num_steps_sampled_before_learning_starts: 10
+        train_batch_size: 256
+        target_network_update_freq: 0
+        min_train_timesteps_per_iteration: 1000
+        optimization:
+          actor_learning_rate: 0.0001
+          critic_learning_rate: 0.0003
+          entropy_learning_rate: 0.0001
+        num_env_runners: 0
+        num_gpus: 1
+        clip_actions: false
+        normalize_actions: true
+        evaluation_interval: 1
+        metrics_num_episodes_for_smoothing: 5
+        # CQL Configs
+        min_q_weight: 5.0
+        bc_iters: 20000
+        temperature: 1.0
+        num_actions: 10
+        lagrangian: False
+        evaluation_config:
+            input: sampler

rllib/examples/_old_api_stack/algorithms/hopper-ppo.yaml

@@ -0,0 +1,17 @@
+# @OldAPIStack
+hopper-ppo:
+    env: Hopper-v1
+    run: PPO
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        gamma: 0.995
+        kl_coeff: 1.0
+        num_epochs: 20
+        lr: .0001
+        minibatch_size: 32768
+        train_batch_size: 160000
+        num_env_runners: 64
+        num_gpus: 4
+        batch_mode: complete_episodes
+        observation_filter: MeanStdFilter

rllib/examples/_old_api_stack/algorithms/humanoid-ppo-gae.yaml

@@ -0,0 +1,24 @@
+# @OldAPIStack
+humanoid-ppo-gae:
+    env: Humanoid-v1
+    run: PPO
+    stop:
+        env_runners/episode_return_mean: 6000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        gamma: 0.995
+        lambda: 0.95
+        clip_param: 0.2
+        kl_coeff: 1.0
+        num_epochs: 20
+        lr: .0001
+        minibatch_size: 32768
+        horizon: 5000
+        train_batch_size: 320000
+        model:
+            free_log_std: true
+        num_env_runners: 64
+        num_gpus: 4
+        batch_mode: complete_episodes
+        observation_filter: MeanStdFilter

rllib/examples/_old_api_stack/algorithms/humanoid-ppo.yaml

@@ -0,0 +1,22 @@
+# @OldAPIStack
+humanoid-ppo:
+    env: Humanoid-v1
+    run: PPO
+    stop:
+        env_runners/episode_return_mean: 6000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        gamma: 0.995
+        kl_coeff: 1.0
+        num_epochs: 20
+        lr: .0001
+        minibatch_size: 32768
+        train_batch_size: 320000
+        model:
+            free_log_std: true
+        use_gae: false
+        num_env_runners: 64
+        num_gpus: 4
+        batch_mode: complete_episodes
+        observation_filter: MeanStdFilter

rllib/examples/_old_api_stack/algorithms/memory-leak-test-appo.yaml

@@ -0,0 +1,15 @@
+# @OldAPIStack
+memory-leak-test-appo:
+    env:
+        ray.rllib.examples.envs.classes.random_env.RandomLargeObsSpaceEnv
+    run: APPO
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        # Switch off np.random, which is known to have memory leaks.
+        env_config:
+            config:
+                static_samples: true
+        num_env_runners: 4
+        num_envs_per_env_runner: 5
+        rollout_fragment_length: 20

rllib/examples/_old_api_stack/algorithms/memory-leak-test-dqn.yaml

@@ -0,0 +1,14 @@
+# @OldAPIStack
+memory-leak-test-dqn:
+    env:
+        ray.rllib.examples.envs.classes.random_env.RandomLargeObsSpaceEnv
+    run: DQN
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        # Switch off np.random, which is known to have memory leaks.
+        env_config:
+            config:
+                static_samples: true
+        replay_buffer_config:
+          capacity: 500  # use small buffer to catch memory leaks

rllib/examples/_old_api_stack/algorithms/memory-leak-test-ppo.yaml

@@ -0,0 +1,17 @@
+# @OldAPIStack
+memory-leak-test-ppo:
+    env:
+        ray.rllib.examples.envs.classes.random_env.RandomLargeObsSpaceEnv
+    run: PPO
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        # Switch off np.random, which is known to have memory leaks.
+        env_config:
+            config:
+                static_samples: true
+        num_env_runners: 4
+        num_envs_per_env_runner: 5
+        train_batch_size: 500
+        minibatch_size: 256
+        num_epochs: 5

rllib/examples/_old_api_stack/algorithms/memory-leak-test-sac.yaml

@@ -0,0 +1,14 @@
+# @OldAPIStack
+memory-leak-test-sac:
+    env:
+        ray.rllib.examples.envs.classes.random_env.RandomLargeObsSpaceEnvContActions
+    run: SAC
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        # Switch off np.random, which is known to have memory leaks.
+        env_config:
+            config:
+                static_samples: true
+        replay_buffer_config:
+          capacity: 500  # use small buffer to catch memory leaks

rllib/examples/_old_api_stack/algorithms/mspacman-sac.yaml

@@ -0,0 +1,45 @@
+# @OldAPIStack
+# Our implementation of SAC discrete can reach up
+# to ~750 reward in 40k timesteps. Run e.g. on a g3.4xlarge with `num_gpus=1`.
+# Uses the hyperparameters published in [2] (see rllib/algorithms/sac/README.md).
+mspacman-sac-tf:
+    env: ale_py:ALE/MsPacman-v5
+    run: SAC
+    stop:
+        env_runners/episode_return_mean: 800
+        timesteps_total: 100000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        env_config:
+            frameskip: 1  # no frameskip
+        gamma: 0.99
+        q_model_config:
+            fcnet_hiddens: [512]
+            fcnet_activation: relu
+        policy_model_config:
+            fcnet_hiddens: [512]
+            fcnet_activation: relu
+        # Do hard syncs.
+        # Soft-syncs seem to work less reliably for discrete action spaces.
+        tau: 1.0
+        target_network_update_freq: 8000
+        # paper uses: 0.98 * -log(1/|A|)
+        target_entropy: 1.755
+        clip_rewards: 1.0
+        n_step: 1
+        rollout_fragment_length: 1
+        train_batch_size: 64
+        min_sample_timesteps_per_iteration: 4
+        # Paper uses 20k random timesteps, which is not exactly the same, but
+        # seems to work nevertheless.
+        replay_buffer_config:
+          type: MultiAgentPrioritizedReplayBuffer
+        num_steps_sampled_before_learning_starts: 20000
+        optimization:
+            actor_learning_rate: 0.0003
+            critic_learning_rate: 0.0003
+            entropy_learning_rate: 0.0003
+        num_env_runners: 0
+        num_gpus: 0
+        metrics_num_episodes_for_smoothing: 5

rllib/examples/_old_api_stack/algorithms/multi-agent-cartpole-crashing-and-stalling-recreate-workers-appo.py

@@ -0,0 +1,77 @@
+# @OldAPIStack
+"""
+Tests, whether APPO can learn in a fault-tolerant fashion in a
+multi-agent setting.
+
+Workers will be configured to automatically get recreated upon failures (here: within
+the environment).
+The environment we use here is configured to crash with a certain probability on each
+`step()` and/or `reset()` call.
+"""
+from ray import tune
+from ray.rllib.algorithms.appo import APPOConfig
+from ray.rllib.examples.envs.classes.cartpole_crashing import MultiAgentCartPoleCrashing
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    EVALUATION_RESULTS,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+
+tune.register_env("ma_env", lambda cfg: MultiAgentCartPoleCrashing(cfg))
+
+stop = {
+    f"{EVALUATION_RESULTS}/{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 800.0,
+    f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 250000,
+}
+
+config = (
+    APPOConfig()
+    .api_stack(
+        enable_rl_module_and_learner=False,
+        enable_env_runner_and_connector_v2=False,
+    )
+    .environment(
+        "ma_env",
+        env_config={
+            "num_agents": 2,
+            # Crash roughly every 300 ts. This should be ok to measure 180.0
+            # reward (episodes are 200 ts long).
+            "p_crash": 0.00005,  # prob to crash during step()
+            "p_crash_reset": 0.0005,  # prob to crash during reset()
+            "init_time_s": 2.0,
+            "p_stall": 0.001,  # prob to stall during step()
+            "p_stall_reset": 0.001,  # prob to stall during reset()
+            "stall_time_sec": (2, 5),  # stall between 2 and 10sec.
+            "stall_on_worker_indices": [2, 3],
+        },
+    )
+    .env_runners(
+        num_env_runners=3,
+        num_envs_per_env_runner=1,
+    )
+    # Switch on resiliency (recreate any failed worker).
+    .fault_tolerance(
+        restart_failed_env_runners=True,
+    )
+    .evaluation(
+        evaluation_num_env_runners=1,
+        evaluation_interval=1,
+        evaluation_duration=25,
+        evaluation_duration_unit="episodes",
+        evaluation_parallel_to_training=True,
+        evaluation_config=APPOConfig.overrides(
+            explore=False,
+            env_config={
+                # Make eval workers solid.
+                # This test is to prove that we can learn with crashing envs,
+                # not evaluate with crashing envs.
+                "p_crash": 0.0,
+                "p_crash_reset": 0.0,
+                "init_time_s": 0.0,
+                "p_stall": 0.0,
+                "p_stall_reset": 0.0,
+            },
+        ),
+    )
+)

rllib/examples/_old_api_stack/algorithms/multi-agent-cartpole-crashing-recreate-workers-appo.py

@@ -0,0 +1,70 @@
+# @OldAPIStack
+"""
+Tests, whether APPO can learn in a fault-tolerant fashion in a
+multi-agent setting.
+
+Workers will be configured to automatically get recreated upon failures (here: within
+the environment).
+The environment we use here is configured to crash with a certain probability on each
+`step()` and/or `reset()` call.
+"""
+from ray import tune
+from ray.rllib.algorithms.appo import APPOConfig
+from ray.rllib.examples.envs.classes.cartpole_crashing import MultiAgentCartPoleCrashing
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    EVALUATION_RESULTS,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+
+tune.register_env("ma_env", lambda cfg: MultiAgentCartPoleCrashing(cfg))
+
+stop = {
+    f"{EVALUATION_RESULTS}/{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 800.0,
+    f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 250000,
+}
+
+config = (
+    APPOConfig()
+    .api_stack(
+        enable_rl_module_and_learner=False,
+        enable_env_runner_and_connector_v2=False,
+    )
+    .environment(
+        "ma_env",
+        env_config={
+            "num_agents": 2,
+            # Crash roughly every 300 ts. This should be ok to measure 180.0
+            # reward (episodes are 200 ts long).
+            "p_crash": 0.0005,  # prob to crash during step()
+            "p_crash_reset": 0.005,  # prob to crash during reset()
+        },
+    )
+    .env_runners(
+        num_env_runners=4,
+        num_envs_per_env_runner=1,
+    )
+    # Switch on resiliency (recreate any failed worker).
+    .fault_tolerance(
+        restart_failed_env_runners=True,
+    )
+    .evaluation(
+        evaluation_num_env_runners=1,
+        evaluation_interval=1,
+        evaluation_duration=25,
+        evaluation_duration_unit="episodes",
+        evaluation_parallel_to_training=True,
+        evaluation_config=APPOConfig.overrides(
+            explore=False,
+            env_config={
+                # Make eval workers solid.
+                # This test is to prove that we can learn with crashing envs,
+                # not evaluate with crashing envs.
+                "p_crash": 0.0,
+                "p_crash_reset": 0.0,
+                "init_time_s": 0.0,
+            },
+        ),
+    )
+)

rllib/examples/_old_api_stack/algorithms/multi-agent-cartpole-w-100-policies-appo.py

@@ -0,0 +1,83 @@
+# @OldAPIStack
+import numpy as np
+
+from ray.rllib.algorithms.appo import APPOConfig
+from ray.rllib.examples.envs.classes.multi_agent import MultiAgentCartPole
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EVALUATION_RESULTS,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+from ray.tune.registry import register_env
+
+register_env("multi_cartpole", lambda _: MultiAgentCartPole({"num_agents": 2}))
+
+# Number of policies overall in the PolicyMap.
+num_policies = 20
+# Number of those policies that should be trained. These are a subset of `num_policies`.
+num_trainable = 10
+
+num_envs_per_env_runner = 5
+
+# Define the config as an APPOConfig object.
+config = (
+    APPOConfig()
+    .api_stack(
+        enable_rl_module_and_learner=False,
+        enable_env_runner_and_connector_v2=False,
+    )
+    .environment("multi_cartpole")
+    .env_runners(
+        num_env_runners=4,
+        num_envs_per_env_runner=num_envs_per_env_runner,
+        observation_filter="MeanStdFilter",
+    )
+    .training(
+        model={
+            "fcnet_hiddens": [32],
+            "fcnet_activation": "linear",
+            "vf_share_layers": True,
+        },
+        num_epochs=1,
+        vf_loss_coeff=0.005,
+        vtrace=True,
+    )
+    .multi_agent(
+        # 2 agents per sub-env.
+        # This is to avoid excessive swapping during an episode rollout, since
+        # Policies are only re-picked at the beginning of each episode.
+        policy_map_capacity=2 * num_envs_per_env_runner,
+        policy_states_are_swappable=True,
+        policies={f"pol{i}" for i in range(num_policies)},
+        # Train only the first n policies.
+        policies_to_train=[f"pol{i}" for i in range(num_trainable)],
+        # Pick one trainable and one non-trainable policy per episode.
+        policy_mapping_fn=(
+            lambda aid, eps, worker, **kw: "pol"
+            + str(
+                np.random.randint(0, num_trainable)
+                if aid == 0
+                else np.random.randint(num_trainable, num_policies)
+            )
+        ),
+    )
+    # On the eval track, always let policy 0 play so we get its results in each results
+    # dict.
+    .evaluation(
+        evaluation_config=APPOConfig.overrides(
+            policy_mapping_fn=(
+                lambda aid, eps, worker, **kw: "pol"
+                + str(0 if aid == 0 else np.random.randint(num_trainable, num_policies))
+            ),
+        ),
+        evaluation_num_env_runners=2,
+        evaluation_interval=1,
+        evaluation_parallel_to_training=True,
+    )
+)
+
+# Define some stopping criteria.
+stop = {
+    f"{EVALUATION_RESULTS}/{ENV_RUNNER_RESULTS}/policy_reward_mean/pol0": 50.0,
+    f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 500000,
+}

rllib/examples/_old_api_stack/algorithms/multi_agent_cartpole_appo_old_api_stack.py

@@ -0,0 +1,46 @@
+# @OldAPIStack
+from ray import tune
+from ray.rllib.algorithms.appo import APPOConfig
+from ray.rllib.examples.envs.classes.multi_agent import MultiAgentCartPole
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+
+tune.registry.register_env("env", lambda cfg: MultiAgentCartPole(config=cfg))
+
+
+config = (
+    APPOConfig()
+    .api_stack(
+        enable_rl_module_and_learner=False,
+        enable_env_runner_and_connector_v2=False,
+    )
+    .environment("env", env_config={"num_agents": 4})
+    .env_runners(
+        num_envs_per_env_runner=5,
+        num_env_runners=4,
+        observation_filter="MeanStdFilter",
+    )
+    .resources(num_gpus=1, _fake_gpus=True)
+    .multi_agent(
+        policies=["p0", "p1", "p2", "p3"],
+        policy_mapping_fn=(lambda agent_id, episode, worker, **kwargs: f"p{agent_id}"),
+    )
+    .training(
+        num_epochs=1,
+        vf_loss_coeff=0.005,
+        vtrace=True,
+        model={
+            "fcnet_hiddens": [32],
+            "fcnet_activation": "linear",
+            "vf_share_layers": True,
+        },
+    )
+)
+
+stop = {
+    f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 600,  # 600 / 4 (==num_agents) = 150
+    f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 200000,
+}

rllib/examples/_old_api_stack/algorithms/multi_agent_cartpole_impala_old_api_stack.py

@@ -0,0 +1,47 @@
+# @OldAPIStack
+from ray import tune
+from ray.rllib.algorithms.impala import IMPALAConfig
+from ray.rllib.examples.envs.classes.multi_agent import MultiAgentCartPole
+from ray.rllib.utils.metrics import (
+    ENV_RUNNER_RESULTS,
+    EPISODE_RETURN_MEAN,
+    NUM_ENV_STEPS_SAMPLED_LIFETIME,
+)
+
+tune.registry.register_env("env", lambda cfg: MultiAgentCartPole(config=cfg))
+
+
+config = (
+    IMPALAConfig()
+    .api_stack(
+        enable_rl_module_and_learner=False,
+        enable_env_runner_and_connector_v2=False,
+    )
+    .environment("env", env_config={"num_agents": 4})
+    .env_runners(
+        num_envs_per_env_runner=5,
+        num_env_runners=4,
+        observation_filter="MeanStdFilter",
+    )
+    .resources(num_gpus=1, _fake_gpus=True)
+    .multi_agent(
+        policies=["p0", "p1", "p2", "p3"],
+        policy_mapping_fn=(lambda agent_id, episode, worker, **kwargs: f"p{agent_id}"),
+    )
+    .training(
+        num_epochs=1,
+        vf_loss_coeff=0.005,
+        vtrace=True,
+        model={
+            "fcnet_hiddens": [32],
+            "fcnet_activation": "linear",
+            "vf_share_layers": True,
+        },
+        replay_proportion=0.0,
+    )
+)
+
+stop = {
+    f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 600,  # 600 / 4 (==num_agents) = 150
+    f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}": 200000,
+}

rllib/examples/_old_api_stack/algorithms/pendulum-cql.yaml

@@ -0,0 +1,44 @@
+# @OldAPIStack
+# Given a SAC-generated offline file generated via:
+# rllib train -f examples/algorithms/sac/pendulum-sac.yaml --no-ray-ui
+
+# Pendulum CQL can attain ~ -300 reward in 10k from that file.
+pendulum-cql:
+    env: Pendulum-v1
+    run: CQL
+    stop:
+        evaluation/env_runners/episode_return_mean: -700
+        timesteps_total: 800000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+
+        # Set seed.
+        seed: 0
+
+        # Use one or more offline files or "input: sampler" for online learning.
+        input: 'dataset'
+        input_config:
+            paths: ["offline/tests/data/pendulum/enormous.zip"]
+            format: 'json'
+        # Our input file above comes from an SAC run. Actions in there
+        # are already normalized (produced by SquashedGaussian).
+        actions_in_input_normalized: true
+        clip_actions: true
+
+        twin_q: true
+        train_batch_size: 2000
+        bc_iters: 100
+        num_env_runners: 2
+        min_time_s_per_iteration: 10
+
+        metrics_num_episodes_for_smoothing: 5
+
+        # Evaluate in an actual environment.
+        evaluation_interval: 1
+        evaluation_num_env_runners: 2
+        evaluation_duration: 10
+        evaluation_parallel_to_training: true
+        evaluation_config:
+            input: sampler
+            explore: False

rllib/examples/_old_api_stack/algorithms/pendulum-sac.yaml

@@ -0,0 +1,35 @@
+# @OldAPIStack
+# Pendulum SAC can attain -150+ reward in 6-7k
+# Configurations are the similar to original softlearning/sac codebase
+pendulum-sac:
+    env: Pendulum-v1
+    run: SAC
+    stop:
+        env_runners/episode_return_mean: -250
+        timesteps_total: 10000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        q_model_config:
+          fcnet_activation: relu
+          fcnet_hiddens: [256, 256]
+        policy_model_config:
+          fcnet_activation: relu
+          fcnet_hiddens: [256, 256]
+        tau: 0.005
+        target_entropy: auto
+        n_step: 1
+        rollout_fragment_length: 1
+        train_batch_size: 256
+        target_network_update_freq: 1
+        min_sample_timesteps_per_iteration: 1000
+        replay_buffer_config:
+          type: MultiAgentPrioritizedReplayBuffer
+        num_steps_sampled_before_learning_starts: 256
+        optimization:
+          actor_learning_rate: 0.0003
+          critic_learning_rate: 0.0003
+          entropy_learning_rate: 0.0003
+        num_env_runners: 0
+        num_gpus: 0
+        metrics_num_episodes_for_smoothing: 5

rllib/examples/_old_api_stack/algorithms/pendulum-transformed-actions-ppo.yaml

@@ -0,0 +1,31 @@
+# @OldAPIStack
+# Can expect improvement to -140 reward in ~300-500k timesteps.
+pendulum-ppo:
+    env: ray.rllib.examples.envs.classes.transformed_action_space_env.TransformedActionPendulum
+    run: PPO
+    stop:
+        env_runners/episode_return_mean: -500
+        timesteps_total: 400000
+    config:
+        # Works for both torch and tf.
+        framework: torch
+
+        # Test, whether PPO is able to learn in "distorted" action spaces.
+        env_config:
+            config:
+                low: 300.0
+                high: 500.0
+
+        normalize_actions: true
+        clip_actions: false
+        vf_clip_param: 10.0
+        num_envs_per_env_runner: 20
+        lambda: 0.1
+        gamma: 0.95
+        lr: 0.0003
+        train_batch_size: 512
+        minibatch_size: 64
+        num_epochs: 6
+        observation_filter: MeanStdFilter
+        model:
+            fcnet_activation: relu

rllib/examples/_old_api_stack/algorithms/pendulum-transformed-actions-sac.yaml

@@ -0,0 +1,44 @@
+# @OldAPIStack
+# TransformedActionPendulum SAC can attain -150+ reward in 6-7k
+# Configurations are the similar to original softlearning/sac codebase
+transformed-actions-pendulum-sac-dummy-torch:
+    env: ray.rllib.examples.envs.classes.transformed_action_space_env.TransformedActionPendulum
+    run: SAC
+    stop:
+        env_runners/episode_return_mean: -200
+        timesteps_total: 10000
+    config:
+        # Works for both torch and tf.
+        seed: 42
+        framework: torch
+
+        # Test, whether SAC is able to learn in "distorted" action spaces.
+        env_config:
+          config:
+            low: 300.0
+            high: 500.0
+
+        horizon: 200
+        q_model_config:
+          fcnet_activation: relu
+          fcnet_hiddens: [256, 256]
+        policy_model_config:
+          fcnet_activation: relu
+          fcnet_hiddens: [256, 256]
+        tau: 0.005
+        target_entropy: auto
+        n_step: 1
+        rollout_fragment_length: 1
+        train_batch_size: 256
+        target_network_update_freq: 1
+        min_sample_timesteps_per_iteration: 1000
+        replay_buffer_config:
+          type: MultiAgentPrioritizedReplayBuffer
+        num_steps_sampled_before_learning_starts: 256
+        optimization:
+          actor_learning_rate: 0.0003
+          critic_learning_rate: 0.0003
+          entropy_learning_rate: 0.0003
+        num_env_runners: 0
+        num_gpus: 0
+        metrics_num_episodes_for_smoothing: 5

rllib/examples/_old_api_stack/algorithms/pong-dqn.yaml

@@ -0,0 +1,35 @@
+# @OldAPIStack
+# You can expect ~20 reward within 1.1m timesteps / 2.1 hours on a K80 GPU
+pong-deterministic-dqn:
+    env: ale_py:ALE/Pong-v5
+    run: DQN
+    stop:
+        env_runners/episode_return_mean: 20
+        time_total_s: 7200
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        # Make analogous to old v4 + NoFrameskip.
+        env_config:
+            frameskip: 1
+            full_action_space: false
+            repeat_action_probability: 0.0
+        num_gpus: 1
+        gamma: 0.99
+        lr: .0001
+        replay_buffer_config:
+          type: MultiAgentPrioritizedReplayBuffer
+          capacity: 50000
+        num_steps_sampled_before_learning_starts: 10000
+        rollout_fragment_length: 4
+        train_batch_size: 32
+        exploration_config:
+          epsilon_timesteps: 200000
+          final_epsilon: .01
+        model:
+          grayscale: True
+          zero_mean: False
+          dim: 42
+        # we should set compress_observations to True because few machines
+        # would be able to contain the replay buffers in memory otherwise
+        compress_observations: True

rllib/examples/_old_api_stack/algorithms/pong-impala-fast.yaml

@@ -0,0 +1,25 @@
+# @OldAPIStack
+# This can reach 18-19 reward in ~3 minutes on p3.16xl head w/m4.16xl workers
+#   128 workers -> 3 minutes (best case)
+#    64 workers -> 4 minutes
+#    32 workers -> 7 minutes
+# See also: pong-impala.yaml, pong-impala-vectorized.yaml
+pong-impala-fast:
+    env: ale_py:ALE/Pong-v5
+    run: IMPALA
+    config:
+        # Make analogous to old v4 + NoFrameskip.
+        env_config:
+            frameskip: 1
+            full_action_space: false
+            repeat_action_probability: 0.0
+        rollout_fragment_length: 50
+        train_batch_size: 1000
+        num_env_runners: 128
+        num_envs_per_env_runner: 5
+        broadcast_interval: 5
+        max_sample_requests_in_flight_per_worker: 1
+        num_multi_gpu_tower_stacks: 4
+        num_gpus: 2
+        model:
+          dim: 42

rllib/examples/_old_api_stack/algorithms/pong-impala-vectorized.yaml

@@ -0,0 +1,17 @@
+# @OldAPIStack
+# This can reach 18-19 reward within 10 minutes on a Tesla M60 GPU (e.g., G3 EC2 node)
+# with 32 workers and 10 envs per worker. This is more efficient than the non-vectorized
+# configuration which requires 128 workers to achieve the same performance.
+pong-impala-vectorized:
+    env: ale_py:ALE/Pong-v5
+    run: IMPALA
+    config:
+        # Make analogous to old v4 + NoFrameskip.
+        env_config:
+            frameskip: 1
+            full_action_space: false
+            repeat_action_probability: 0.0
+        rollout_fragment_length: 50
+        train_batch_size: 500
+        num_env_runners: 32
+        num_envs_per_env_runner: 10

rllib/examples/_old_api_stack/algorithms/pong-impala.yaml

@@ -0,0 +1,19 @@
+# @OldAPIStack
+# This can reach 18-19 reward within 10 minutes on a Tesla M60 GPU (e.g., G3 EC2 node):
+#   128 workers -> 8 minutes
+#    32 workers -> 17 minutes
+#    16 workers -> 40 min+
+# See also: pong-impala-fast.yaml, pong-impala-vectorized.yaml
+pong-impala:
+    env: ale_py:ALE/Pong-v5
+    run: IMPALA
+    config:
+        # Make analogous to old v4 + NoFrameskip.
+        env_config:
+            frameskip: 1
+            full_action_space: false
+            repeat_action_probability: 0.0
+        rollout_fragment_length: 50
+        train_batch_size: 500
+        num_env_runners: 128
+        num_envs_per_env_runner: 1

rllib/examples/_old_api_stack/algorithms/pong-rainbow.yaml

@@ -0,0 +1,37 @@
+# @OldAPIStack
+pong-deterministic-rainbow:
+    env: ale_py:ALE/Pong-v5
+    run: DQN
+    stop:
+        env_runners/episode_return_mean: 20
+    config:
+        # Make analogous to old v4 + NoFrameskip.
+        env_config:
+            frameskip: 1
+            full_action_space: false
+            repeat_action_probability: 0.0
+        num_atoms: 51
+        noisy: True
+        gamma: 0.99
+        lr: .0001
+        hiddens: [512]
+        rollout_fragment_length: 4
+        train_batch_size: 32
+        exploration_config:
+          epsilon_timesteps: 2
+          final_epsilon: 0.0
+        target_network_update_freq: 500
+        replay_buffer_config:
+          type: MultiAgentPrioritizedReplayBuffer
+          prioritized_replay_alpha: 0.5
+          capacity: 50000
+        num_steps_sampled_before_learning_starts: 10000
+        n_step: 3
+        gpu: True
+        model:
+          grayscale: True
+          zero_mean: False
+          dim: 42
+        # we should set compress_observations to True because few machines
+        # would be able to contain the replay buffers in memory otherwise
+        compress_observations: True

rllib/examples/_old_api_stack/algorithms/walker2d-ppo.yaml

@@ -0,0 +1,16 @@
+# @OldAPIStack
+walker2d-v1-ppo:
+    env: Walker2d-v1
+    run: PPO
+    config:
+        # Works for both torch and tf.
+        framework: torch
+        kl_coeff: 1.0
+        num_epochs: 20
+        lr: .0001
+        minibatch_size: 32768
+        train_batch_size: 320000
+        num_env_runners: 64
+        num_gpus: 4
+        batch_mode: complete_episodes
+        observation_filter: MeanStdFilter