chore: import upstream snapshot with attribution

doc/source/rllib/doc_code/advanced_api.py

@@ -0,0 +1,125 @@
+# flake8: noqa
+
+# __rllib-adv_api_counter_begin__
+import ray
+
+
+@ray.remote
+class Counter:
+    def __init__(self):
+        self.count = 0
+
+    def inc(self, n):
+        self.count += n
+
+    def get(self):
+        return self.count
+
+
+# on the driver
+counter = Counter.options(name="global_counter").remote()
+print(ray.get(counter.get.remote()))  # get the latest count
+
+# in your envs
+counter = ray.get_actor("global_counter")
+counter.inc.remote(1)  # async call to increment the global count
+# __rllib-adv_api_counter_end__
+
+
+# __rllib-adv_api_explore_begin__
+from ray.rllib.algorithms.algorithm_config import AlgorithmConfig
+
+config = AlgorithmConfig().env_runners(
+    exploration_config={
+        # Special `type` key provides class information
+        "type": "StochasticSampling",
+        # Add any needed constructor args here.
+        "constructor_arg": "value",
+    }
+)
+# __rllib-adv_api_explore_end__
+
+
+# __rllib-adv_api_evaluation_1_begin__
+from ray.rllib.algorithms.algorithm_config import AlgorithmConfig
+
+# Run one evaluation step on every 3rd `Algorithm.train()` call.
+config = AlgorithmConfig().evaluation(
+    evaluation_interval=3,
+)
+# __rllib-adv_api_evaluation_1_end__
+
+
+# __rllib-adv_api_evaluation_2_begin__
+# Every time we run an evaluation step, run it for exactly 10 episodes.
+config = AlgorithmConfig().evaluation(
+    evaluation_duration=10,
+    evaluation_duration_unit="episodes",
+)
+# Every time we run an evaluation step, run it for (close to) 200 timesteps.
+config = AlgorithmConfig().evaluation(
+    evaluation_duration=200,
+    evaluation_duration_unit="timesteps",
+)
+# __rllib-adv_api_evaluation_2_end__
+
+
+# __rllib-adv_api_evaluation_3_begin__
+# Every time we run an evaluation step, run it for exactly 10 episodes, no matter,
+# how many eval workers we have.
+config = AlgorithmConfig().evaluation(
+    evaluation_duration=10,
+    evaluation_duration_unit="episodes",
+    # What if number of eval workers is non-dividable by 10?
+    # -> Run 7 episodes (1 per eval worker), then run 3 more episodes only using
+    #    evaluation workers 1-3 (evaluation workers 4-7 remain idle during that time).
+    evaluation_num_env_runners=7,
+)
+# __rllib-adv_api_evaluation_3_end__
+
+
+# __rllib-adv_api_evaluation_4_begin__
+# Run evaluation and training at the same time via threading and make sure they roughly
+# take the same time, such that the next `Algorithm.train()` call can execute
+# immediately and not have to wait for a still ongoing (e.g. b/c of very long episodes)
+# evaluation step:
+config = AlgorithmConfig().evaluation(
+    evaluation_interval=2,
+    # run evaluation and training in parallel
+    evaluation_parallel_to_training=True,
+    # automatically end evaluation when train step has finished
+    evaluation_duration="auto",
+    evaluation_duration_unit="timesteps",  # <- this setting is ignored; RLlib
+    # will always run by timesteps (not by complete
+    # episodes) in this duration=auto mode
+)
+# __rllib-adv_api_evaluation_4_end__
+
+
+# __rllib-adv_api_evaluation_5_begin__
+# Switching off exploration behavior for evaluation workers
+# (see rllib/algorithms/algorithm.py). Use any keys in this sub-dict that are
+# also supported in the main Algorithm config.
+config = AlgorithmConfig().evaluation(
+    evaluation_config=AlgorithmConfig.overrides(explore=False),
+)
+# ... which is a more type-checked version of the old-style:
+# config = AlgorithmConfig().evaluation(
+#    evaluation_config={"explore": False},
+# )
+# __rllib-adv_api_evaluation_5_end__
+
+
+# __rllib-adv_api_evaluation_6_begin__
+# Having an environment that occasionally blocks completely for e.g. 10min would
+# also affect (and block) training. Here is how you can defend your evaluation setup
+# against oft-crashing or -stalling envs (or other unstable components on your evaluation
+# workers).
+config = AlgorithmConfig().evaluation(
+    evaluation_interval=1,
+    evaluation_parallel_to_training=True,
+    evaluation_duration="auto",
+    evaluation_duration_unit="timesteps",  # <- default anyway
+    evaluation_force_reset_envs_before_iteration=True,  # <- default anyway
+)
+# __rllib-adv_api_evaluation_6_end__

doc/source/rllib/doc_code/custom_gym_env.py

@@ -0,0 +1,40 @@
+# __rllib-custom-gym-env-begin__
+import gymnasium as gym
+import numpy as np
+
+import ray
+from ray.rllib.algorithms.ppo import PPOConfig
+
+
+class SimpleCorridor(gym.Env):
+    def __init__(self, config):
+        self.end_pos = config["corridor_length"]
+        self.cur_pos = 0.0
+        self.action_space = gym.spaces.Discrete(2)  # right/left
+        self.observation_space = gym.spaces.Box(0.0, self.end_pos, shape=(1,))
+
+    def reset(self, *, seed=None, options=None):
+        self.cur_pos = 0.0
+        return np.array([self.cur_pos]), {}
+
+    def step(self, action):
+        if action == 0 and self.cur_pos > 0.0:  # move right (towards goal)
+            self.cur_pos -= 1.0
+        elif action == 1:  # move left (towards start)
+            self.cur_pos += 1.0
+        if self.cur_pos >= self.end_pos:
+            return np.array([0.0]), 1.0, True, True, {}
+        else:
+            return np.array([self.cur_pos]), -0.1, False, False, {}
+
+
+ray.init()
+
+config = PPOConfig().environment(SimpleCorridor, env_config={"corridor_length": 5})
+algo = config.build()
+
+for _ in range(3):
+    print(algo.train())
+
+algo.stop()
+# __rllib-custom-gym-env-end__

doc/source/rllib/doc_code/dreamerv3_inference.py

@@ -0,0 +1,67 @@
+import gymnasium as gym
+import numpy as np
+
+import tree  # pip install dm_tree
+
+from ray.rllib.algorithms.dreamerv3.dreamerv3 import DreamerV3Config
+from ray.rllib.core.columns import Columns
+from ray.rllib.utils.framework import convert_to_tensor
+
+
+env_name = "CartPole-v1"
+# Use the vector env API.
+env = gym.make_vec(env_name, num_envs=1, vectorization_mode="sync")
+
+terminated = truncated = False
+# Reset the env.
+obs, _ = env.reset()
+# Every time, we start a new episode, we should set is_first to True for the upcoming
+# action inference.
+is_first = 1.0
+
+# Create the algorithm from a simple config.
+config = (
+    DreamerV3Config()
+    .environment("CartPole-v1")
+    .training(model_size="XS", training_ratio=1024)
+)
+algo = config.build()
+
+# Extract the actual RLModule from the local (Dreamer) EnvRunner.
+rl_module = algo.env_runner.module
+# Get initial states from RLModule (note that these are always B=1, so this matches
+# our num_envs=1; if you are using a vector env >1, you would have to repeat the
+# returned states `num_env` times to get the correct batch size):
+states = rl_module.get_initial_state()
+# Batch the states to B=1.
+states = tree.map_structure(lambda s: s.unsqueeze(0), states)
+
+while not terminated and not truncated:
+    # Use the RLModule for action computations directly.
+    # DreamerV3 expects this particular batch format:
+    # obs=[B, T, ...]
+    # prev. states=[B, ...]
+    # `is_first`=[B]
+    batch = {
+        # States is already batched (see above).
+        Columns.STATE_IN: states,
+        # `obs` is already batched (due to vector env), but needs time-rank.
+        Columns.OBS: convert_to_tensor(obs, framework="torch")[None],
+        # Set to True at beginning of episode.
+        "is_first": convert_to_tensor(is_first, "torch")[None],
+    }
+    outs = rl_module.forward_inference(batch)
+    # Alternatively, call `forward_exploration` in case you want stochastic, non-greedy
+    # actions.
+    # outs = rl_module.forward_exploration(batch)
+
+    # Extract actions (remove time-rank) from outs.
+    actions = outs[Columns.ACTIONS].numpy()[0]
+    # Extract states from out. States are returned as batched.
+    states = outs[Columns.STATE_OUT]
+
+    # Perform a step in the env. Note that actions are still batched, which
+    # is ok, because we have a vector env.
+    obs, reward, terminated, truncated, info = env.step(actions)
+    # Not at the beginning of the episode anymore.
+    is_first = 0.0

doc/source/rllib/doc_code/replay_buffer_demo.py

@@ -0,0 +1,175 @@
+# Demonstration of RLlib's ReplayBuffer workflow
+
+from typing import Optional
+import random
+import numpy as np
+
+from ray import tune
+from ray.rllib.utils.replay_buffers import ReplayBuffer, StorageUnit
+from ray.rllib.utils.annotations import override
+from ray.rllib.utils.typing import SampleBatchType
+from ray.rllib.utils.replay_buffers.utils import validate_buffer_config
+from ray.rllib.examples.envs.classes.random_env import RandomEnv
+from ray.rllib.policy.sample_batch import SampleBatch, concat_samples
+from ray.rllib.algorithms.dqn.dqn import DQNConfig
+
+
+# __sphinx_doc_replay_buffer_type_specification__begin__
+config = (
+    DQNConfig()
+    .api_stack(
+        enable_env_runner_and_connector_v2=False, enable_rl_module_and_learner=False
+    )
+    .training(replay_buffer_config={"type": ReplayBuffer})
+)
+
+another_config = (
+    DQNConfig()
+    .api_stack(
+        enable_env_runner_and_connector_v2=False, enable_rl_module_and_learner=False
+    )
+    .training(replay_buffer_config={"type": "ReplayBuffer"})
+)
+
+
+yet_another_config = (
+    DQNConfig()
+    .api_stack(
+        enable_env_runner_and_connector_v2=False, enable_rl_module_and_learner=False
+    )
+    .training(
+        replay_buffer_config={"type": "ray.rllib.utils.replay_buffers.ReplayBuffer"}
+    )
+)
+
+validate_buffer_config(config)
+validate_buffer_config(another_config)
+validate_buffer_config(yet_another_config)
+
+# After validation, all three configs yield the same effective config
+assert (
+    config.replay_buffer_config
+    == another_config.replay_buffer_config
+    == yet_another_config.replay_buffer_config
+)
+
+# __sphinx_doc_replay_buffer_type_specification__end__
+
+
+# __sphinx_doc_replay_buffer_basic_interaction__begin__
+# We choose fragments because it does not impose restrictions on our batch to be added
+buffer = ReplayBuffer(capacity=2, storage_unit=StorageUnit.FRAGMENTS)
+dummy_batch = SampleBatch({"a": [1], "b": [2]})
+buffer.add(dummy_batch)
+buffer.sample(2)
+# Because elements can be sampled multiple times, we receive a concatenated version
+# of dummy_batch `{a: [1, 1], b: [2, 2,]}`.
+# __sphinx_doc_replay_buffer_basic_interaction__end__
+
+
+# __sphinx_doc_replay_buffer_own_buffer__begin__
+class LessSampledReplayBuffer(ReplayBuffer):
+    @override(ReplayBuffer)
+    def sample(
+        self, num_items: int, evict_sampled_more_then: int = 30, **kwargs
+    ) -> Optional[SampleBatchType]:
+        """Evicts experiences that have been sampled > evict_sampled_more_then times."""
+        idxes = [random.randint(0, len(self) - 1) for _ in range(num_items)]
+        often_sampled_idxes = list(
+            filter(lambda x: self._hit_count[x] >= evict_sampled_more_then, set(idxes))
+        )
+
+        sample = self._encode_sample(idxes)
+        self._num_timesteps_sampled += sample.count
+
+        for idx in often_sampled_idxes:
+            del self._storage[idx]
+            self._hit_count = np.append(
+                self._hit_count[:idx], self._hit_count[idx + 1 :]
+            )
+
+        return sample
+
+
+config = (
+    DQNConfig()
+    .api_stack(
+        enable_env_runner_and_connector_v2=False, enable_rl_module_and_learner=False
+    )
+    .environment(env="CartPole-v1")
+    .training(replay_buffer_config={"type": LessSampledReplayBuffer})
+)
+
+tune.Tuner(
+    "DQN",
+    param_space=config,
+    run_config=tune.RunConfig(
+        stop={"training_iteration": 1},
+    ),
+).fit()
+# __sphinx_doc_replay_buffer_own_buffer__end__
+
+# __sphinx_doc_replay_buffer_advanced_usage_storage_unit__begin__
+# This line will make our buffer store only complete episodes found in a batch
+config.training(replay_buffer_config={"storage_unit": StorageUnit.EPISODES})
+
+less_sampled_buffer = LessSampledReplayBuffer(**config.replay_buffer_config)
+
+# Gather some random experiences
+env = RandomEnv()
+terminated = truncated = False
+batch = SampleBatch({})
+t = 0
+while not terminated and not truncated:
+    obs, reward, terminated, truncated, info = env.step([0, 0])
+    # Note that in order for RLlib to find out about start and end of an episode,
+    # "t" and "terminateds" have to properly mark an episode's trajectory
+    one_step_batch = SampleBatch(
+        {
+            "obs": [obs],
+            "t": [t],
+            "reward": [reward],
+            "terminateds": [terminated],
+            "truncateds": [truncated],
+        }
+    )
+    batch = concat_samples([batch, one_step_batch])
+    t += 1
+
+less_sampled_buffer.add(batch)
+for i in range(10):
+    assert len(less_sampled_buffer._storage) == 1
+    less_sampled_buffer.sample(num_items=1, evict_sampled_more_then=9)
+
+assert len(less_sampled_buffer._storage) == 0
+# __sphinx_doc_replay_buffer_advanced_usage_storage_unit__end__
+
+
+# __sphinx_doc_replay_buffer_advanced_usage_underlying_buffers__begin__
+config = (
+    DQNConfig()
+    .api_stack(
+        enable_env_runner_and_connector_v2=False, enable_rl_module_and_learner=False
+    )
+    .training(
+        replay_buffer_config={
+            "type": "MultiAgentReplayBuffer",
+            "underlying_replay_buffer_config": {
+                "type": LessSampledReplayBuffer,
+                # We can specify the default call argument
+                # for the sample method of the underlying buffer method here.
+                "evict_sampled_more_then": 20,
+            },
+        }
+    )
+    .environment(env="CartPole-v1")
+)
+
+tune.Tuner(
+    "DQN",
+    param_space=config.to_dict(),
+    run_config=tune.RunConfig(
+        stop={"env_runners/episode_return_mean": 40, "training_iteration": 7},
+    ),
+).fit()
+# __sphinx_doc_replay_buffer_advanced_usage_underlying_buffers__end__

doc/source/rllib/doc_code/rllib_on_ray_readme.py

@@ -0,0 +1,170 @@
+# __quick_start_begin__
+import gymnasium as gym
+import numpy as np
+import torch
+from typing import Dict, Tuple, Any, Optional
+
+from ray.rllib.algorithms.ppo import PPOConfig
+
+
+# Define your problem using python and Farama-Foundation's gymnasium API:
+class SimpleCorridor(gym.Env):
+    """Corridor environment where an agent must learn to move right to reach the exit.
+
+    ---------------------
+    | S | 1 | 2 | 3 | G |   S=start; G=goal; corridor_length=5
+    ---------------------
+
+    Actions:
+        0: Move left
+        1: Move right
+
+    Observations:
+        A single float representing the agent's current position (index)
+        starting at 0.0 and ending at corridor_length
+
+    Rewards:
+        -0.1 for each step
+        +1.0 when reaching the goal
+
+    Episode termination:
+        When the agent reaches the goal (position >= corridor_length)
+    """
+
+    def __init__(self, config):
+        self.end_pos = config["corridor_length"]
+        self.cur_pos = 0.0
+        self.action_space = gym.spaces.Discrete(2)  # 0=left, 1=right
+        self.observation_space = gym.spaces.Box(0.0, self.end_pos, (1,), np.float32)
+
+    def reset(
+        self, *, seed: Optional[int] = None, options: Optional[Dict] = None
+    ) -> Tuple[np.ndarray, Dict]:
+        """Reset the environment for a new episode.
+
+        Args:
+            seed: Random seed for reproducibility
+            options: Additional options (not used in this environment)
+
+        Returns:
+            Initial observation of the new episode and an info dict.
+        """
+        super().reset(seed=seed)  # Initialize RNG if seed is provided
+        self.cur_pos = 0.0
+        # Return initial observation.
+        return np.array([self.cur_pos], np.float32), {}
+
+    def step(self, action: int) -> Tuple[np.ndarray, float, bool, bool, Dict]:
+        """Take a single step in the environment based on the provided action.
+
+        Args:
+            action: 0 for left, 1 for right
+
+        Returns:
+            A tuple of (observation, reward, terminated, truncated, info):
+                observation: Agent's new position
+                reward: Reward from taking the action (-0.1 or +1.0)
+                terminated: Whether episode is done (reached goal)
+                truncated: Whether episode was truncated (always False here)
+                info: Additional information (empty dict)
+        """
+        # Walk left if action is 0 and we're not at the leftmost position
+        if action == 0 and self.cur_pos > 0:
+            self.cur_pos -= 1
+        # Walk right if action is 1
+        elif action == 1:
+            self.cur_pos += 1
+        # Set `terminated` flag when end of corridor (goal) reached.
+        terminated = self.cur_pos >= self.end_pos
+        truncated = False
+        # +1 when goal reached, otherwise -0.1.
+        reward = 1.0 if terminated else -0.1
+        return np.array([self.cur_pos], np.float32), reward, terminated, truncated, {}
+
+
+# Create an RLlib Algorithm instance from a PPOConfig object.
+print("Setting up the PPO configuration...")
+config = (
+    PPOConfig().environment(
+        # Env class to use (our custom gymnasium environment).
+        SimpleCorridor,
+        # Config dict passed to our custom env's constructor.
+        # Use corridor with 20 fields (including start and goal).
+        env_config={"corridor_length": 20},
+    )
+    # Parallelize environment rollouts for faster training.
+    .env_runners(num_env_runners=3)
+    # Use a smaller network for this simple task
+    .training(model={"fcnet_hiddens": [64, 64]})
+)
+
+# Construct the actual PPO algorithm object from the config.
+algo = config.build_algo()
+rl_module = algo.get_module()
+
+# Train for n iterations and report results (mean episode rewards).
+# Optimal reward calculation:
+# - Need at least 19 steps to reach the goal (from position 0 to 19)
+# - Each step (except last) gets -0.1 reward: 18 * (-0.1) = -1.8
+# - Final step gets +1.0 reward
+# - Total optimal reward: -1.8 + 1.0 = -0.8
+print("\nStarting training loop...")
+for i in range(5):
+    results = algo.train()
+
+    # Log the metrics from training results
+    print(f"Iteration {i+1}")
+    print(f"  Training metrics: {results['env_runners']}")
+
+# Save the trained algorithm (optional)
+checkpoint_dir = algo.save()
+print(f"\nSaved model checkpoint to: {checkpoint_dir}")
+
+print("\nRunning inference with the trained policy...")
+# Create a test environment with a shorter corridor to verify the agent's behavior
+env = SimpleCorridor({"corridor_length": 10})
+# Get the initial observation (should be: [0.0] for the starting position).
+obs, info = env.reset()
+terminated = truncated = False
+total_reward = 0.0
+step_count = 0
+
+# Play one episode and track the agent's trajectory
+print("\nAgent trajectory:")
+positions = [float(obs[0])]  # Track positions for visualization
+
+while not terminated and not truncated and step_count < 1000:
+    # Compute an action given the current observation
+    action_logits = rl_module.forward_inference(
+        {"obs": torch.from_numpy(obs).unsqueeze(0)}
+    )["action_dist_inputs"].numpy()[
+        0
+    ]  # [0]: Batch dimension=1
+
+    # Get the action with highest probability
+    action = np.argmax(action_logits)
+
+    # Log the agent's decision
+    action_name = "LEFT" if action == 0 else "RIGHT"
+    print(f"  Step {step_count}: Position {obs[0]:.1f}, Action: {action_name}")
+
+    # Apply the computed action in the environment
+    obs, reward, terminated, truncated, info = env.step(action)
+    positions.append(float(obs[0]))
+
+    # Sum up rewards
+    total_reward += reward
+    step_count += 1
+
+# Report final results
+print(f"\nEpisode complete:")
+print(f"  Steps taken: {step_count}")
+print(f"  Total reward: {total_reward:.2f}")
+print(f"  Final position: {obs[0]:.1f}")
+
+# Verify the agent has learned the optimal policy
+if total_reward > -0.5 and obs[0] >= 9.0:
+    print("  Success! The agent has learned the optimal policy (always move right).")
+else:
+    print("  Failure! The agent didn't reach the goal within 1000 timesteps.")
+# __quick_start_end__

doc/source/rllib/doc_code/sa_episode.py

@@ -0,0 +1,270 @@
+# flake8: noqa
+import copy
+
+# __rllib-sa-episode-01-begin__
+from ray.rllib.env.single_agent_episode import SingleAgentEpisode
+
+# Construct a new episode (without any data in it yet).
+episode = SingleAgentEpisode()
+assert len(episode) == 0
+
+episode.add_env_reset(observation="obs_0", infos="info_0")
+# Even with the initial obs/infos, the episode is still considered len=0.
+assert len(episode) == 0
+
+# Fill the episode with some fake data (5 timesteps).
+for i in range(5):
+    episode.add_env_step(
+        observation=f"obs_{i+1}",
+        action=f"act_{i}",
+        reward=f"rew_{i}",
+        terminated=False,
+        truncated=False,
+        infos=f"info_{i+1}",
+    )
+assert len(episode) == 5
+# __rllib-sa-episode-01-end__
+
+
+# __rllib-sa-episode-02-begin__
+# We can now access information from the episode via its getter APIs.
+
+from ray.rllib.utils.test_utils import check
+
+# Get the very first observation ("reset observation"). Note that a single observation
+# is returned here (not a list of size 1 or a batch of size 1).
+check(episode.get_observations(0), "obs_0")
+# ... which is the same as using the indexing operator on the Episode's
+# `observations` property:
+check(episode.observations[0], "obs_0")
+
+# You can also get several observations at once by providing a list of indices:
+check(episode.get_observations([1, 2]), ["obs_1", "obs_2"])
+# .. or a slice of observations by providing a python slice object:
+check(episode.get_observations(slice(1, 3)), ["obs_1", "obs_2"])
+
+# Note that when passing only a single index, a single item is returned.
+# Whereas when passing a list of indices or a slice, a list of items is returned.
+
+# Similarly for getting rewards:
+# Get the last reward.
+check(episode.get_rewards(-1), "rew_4")
+# ... which is the same as using the slice operator on the `rewards` property:
+check(episode.rewards[-1], "rew_4")
+
+# Similarly for getting actions:
+# Get the first action in the episode (single item, not batched).
+# This works regardless of the action space.
+check(episode.get_actions(0), "act_0")
+# ... which is the same as using the indexing operator on the `actions` property:
+check(episode.actions[0], "act_0")
+
+# Finally, you can slice the entire episode using the []-operator with a slice notation:
+sliced_episode = episode[3:4]
+check(list(sliced_episode.observations), ["obs_3", "obs_4"])
+check(list(sliced_episode.actions), ["act_3"])
+check(list(sliced_episode.rewards), ["rew_3"])
+
+# __rllib-sa-episode-02-end__
+
+import copy  # noqa
+
+episode_2 = copy.deepcopy(episode)
+
+# __rllib-sa-episode-03-begin__
+
+# Episodes start in the non-numpy'ized state (in which data is stored
+# under the hood in lists).
+assert episode.is_numpy is False
+
+# Call `to_numpy()` to convert all stored data from lists of individual (possibly
+# complex) items to numpy arrays. Note that RLlib normally performs this method call,
+# so users don't need to call `to_numpy()` themselves.
+episode.to_numpy()
+assert episode.is_numpy is True
+
+# __rllib-sa-episode-03-end__
+
+episode = episode_2
+
+# __rllib-sa-episode-04-begin__
+
+# An ongoing episode (of length 5):
+assert len(episode) == 5
+assert episode.is_done is False
+
+# During an `EnvRunner.sample()` rollout, when enough data has been collected into
+# one or more Episodes, the `EnvRunner` calls the `cut()` method, interrupting
+# the ongoing Episode and returning a new continuation chunk (with which the
+# `EnvRunner` can continue collecting data during the next call to `sample()`):
+continuation_episode = episode.cut()
+
+# The length is still 5, but the length of the continuation chunk is 0.
+assert len(episode) == 5
+assert len(continuation_episode) == 0
+
+# Thanks to the lookback buffer, we can still access the most recent observation
+# in the continuation chunk:
+check(continuation_episode.get_observations(-1), "obs_5")
+
+# __rllib-sa-episode-04-end__
+
+
+# __rllib-sa-episode-05-begin__
+
+# Construct a new episode (with some data in its lookback buffer).
+episode = SingleAgentEpisode(
+    observations=["o0", "o1", "o2", "o3"],
+    actions=["a0", "a1", "a2"],
+    rewards=[0.0, 1.0, 2.0],
+    len_lookback_buffer=3,
+)
+# Since our lookback buffer is 3, all data already specified in the constructor should
+# now be in the lookback buffer (and not be part of the `episode` chunk), meaning
+# the length of `episode` should still be 0.
+assert len(episode) == 0
+
+# .. and trying to get the first reward will hence lead to an IndexError.
+try:
+    episode.get_rewards(0)
+except IndexError:
+    pass
+
+# Get the last 3 rewards (using the lookback buffer).
+check(episode.get_rewards(slice(-3, None)), [0.0, 1.0, 2.0])
+
+# Assuming the episode actually started with `obs_0` (reset obs),
+# then `obs_1` + `act_0` + reward=0.0, but your model always requires a 1D reward tensor
+# of shape (5,) with the 5 most recent rewards in it.
+# You could try to code for this by manually filling the missing 2 timesteps with zeros:
+last_5_rewards = [0.0, 0.0] + episode.get_rewards(slice(-3, None))
+# However, this will become extremely tedious, especially when moving to (possibly more
+# complex) observations and actions.
+
+# Instead, `SingleAgentEpisode` getters offer some useful options to solve this problem:
+last_5_rewards = episode.get_rewards(slice(-5, None), fill=0.0)
+# Note that the `fill` argument allows you to even go further back into the past, provided
+# you are ok with filling timesteps that are not covered by the lookback buffer with
+# a fixed value.
+
+# __rllib-sa-episode-05-end__
+
+
+# __rllib-sa-episode-06-begin__
+
+# Construct a new episode (len=3 and lookback buffer=3).
+episode = SingleAgentEpisode(
+    observations=[
+        "o-3",
+        "o-2",
+        "o-1",  # <- lookback  # noqa
+        "o0",
+        "o1",
+        "o2",
+        "o3",  # <- actual episode data  # noqa
+    ],
+    actions=[
+        "a-3",
+        "a-2",
+        "a-1",  # <- lookback  # noqa
+        "a0",
+        "a1",
+        "a2",  # <- actual episode data  # noqa
+    ],
+    rewards=[
+        -3.0,
+        -2.0,
+        -1.0,  # <- lookback  # noqa
+        0.0,
+        1.0,
+        2.0,  # <- actual episode data  # noqa
+    ],
+    len_lookback_buffer=3,
+)
+assert len(episode) == 3
+
+# In case you want to loop through global timesteps 0 to 2 (timesteps -3, -2, and -1
+# being the lookback buffer) and at each such global timestep look 2 timesteps back,
+# you can do so easily using the `neg_index_as_lookback` arg like so:
+for global_ts in [0, 1, 2]:
+    rewards = episode.get_rewards(
+        slice(global_ts - 2, global_ts + 1),
+        # Switch behavior of negative indices from "from-the-end" to
+        # "into the lookback buffer":
+        neg_index_as_lookback=True,
+    )
+    print(rewards)
+
+# The expected output should be:
+# [-2.0, -1.0, 0.0]  # global ts=0 (plus looking back 2 ts)
+# [-1.0, 0.0, 1.0]   # global ts=1 (plus looking back 2 ts)
+# [0.0, 1.0, 2.0]    # global ts=2 (plus looking back 2 ts)
+
+# __rllib-sa-episode-06-end__
+
+
+# Looking back from ts=1, get the previous 4 rewards AND fill with 0.0
+# in case we go over the beginning (ts=0). So we would expect
+# [0.0, 0.0, 0.0, r0] to be returned here, where r0 is the very first received
+# reward in the episode:
+episode.get_rewards(slice(-4, 0), neg_index_as_lookback=True, fill=0.0)
+
+# Note the use of fill=0.0 here (fill everything that's out of range with this
+# value) AND the argument `neg_index_as_lookback=True`, which interprets
+# negative indices as being left of ts=0 (e.g. -1 being the timestep before
+# ts=0).
+
+import gymnasium as gym
+import numpy as np
+
+# Assuming we had a complex action space (nested gym.spaces.Dict) with one or
+# more elements being Discrete or MultiDiscrete spaces:
+# 1) The `fill=...` argument would still work, filling all spaces (Boxes,
+# Discrete) with that provided value.
+# 2) Setting the flag `one_hot_discrete=True` would convert those discrete
+# sub-components automatically into one-hot (or multi-one-hot) tensors.
+# This simplifies the task of having to provide the previous 4 (nested and
+# partially discrete/multi-discrete) actions for each timestep within a training
+# batch, thereby filling timesteps before the episode started with 0.0s and
+# one-hot'ing the discrete/multi-discrete components in these actions:
+episode = SingleAgentEpisode(
+    action_space=gym.spaces.Dict(
+        {
+            "a": gym.spaces.Discrete(3),
+            "b": gym.spaces.MultiDiscrete([2, 3]),
+            "c": gym.spaces.Box(-1.0, 1.0, (2,)),
+        }
+    )
+)
+
+# ... fill episode with data ...
+episode.add_env_reset(observation=0)
+# ... from a few steps.
+episode.add_env_step(
+    observation=1,
+    action={"a": 0, "b": np.array([1, 2]), "c": np.array([0.5, -0.5], np.float32)},
+    reward=1.0,
+)
+
+# In your connector
+prev_4_a = []
+# Note here that len(episode) does NOT include the lookback buffer.
+for ts in range(len(episode)):
+    prev_4_a.append(
+        episode.get_actions(
+            indices=slice(ts - 4, ts),
+            # Make sure negative indices are interpreted as
+            # "into lookback buffer"
+            neg_index_as_lookback=True,
+            # Zero-out everything even further before the lookback buffer.
+            fill=0.0,
+            # Take care of discrete components (get ready as NN input).
+            one_hot_discrete=True,
+        )
+    )
+
+# Finally, convert from list of batch items to a struct (same as action space)
+# of batched (numpy) arrays, in which all leafs have B==len(prev_4_a).
+from ray.rllib.utils.spaces.space_utils import batch
+
+prev_4_actions_col = batch(prev_4_a)