chore: import upstream snapshot with attribution

rllib/examples/multi_agent/utils/__init__.py

@@ -0,0 +1,43 @@
+import sys
+
+from ray.rllib.examples.multi_agent.utils.self_play_callback import SelfPlayCallback
+from ray.rllib.examples.multi_agent.utils.self_play_callback_old_api_stack import (
+    SelfPlayCallbackOldAPIStack,
+)
+from ray.rllib.examples.multi_agent.utils.self_play_league_based_callback import (
+    SelfPlayLeagueBasedCallback,
+)
+from ray.rllib.examples.multi_agent.utils.self_play_league_based_callback_old_api_stack import (  # noqa
+    SelfPlayLeagueBasedCallbackOldAPIStack,
+)
+
+
+def ask_user_for_action(time_step):
+    """Asks the user for a valid action on the command line and returns it.
+
+    Re-queries the user until she picks a valid one.
+
+    Args:
+        time_step: The open spiel Environment time-step object.
+    """
+    pid = time_step.observations["current_player"]
+    legal_moves = time_step.observations["legal_actions"][pid]
+    choice = -1
+    while choice not in legal_moves:
+        print("Choose an action from {}:".format(legal_moves))
+        sys.stdout.flush()
+        choice_str = input()
+        try:
+            choice = int(choice_str)
+        except ValueError:
+            continue
+    return choice
+
+
+__all__ = [
+    "ask_user_for_action",
+    "SelfPlayCallback",
+    "SelfPlayLeagueBasedCallback",
+    "SelfPlayCallbackOldAPIStack",
+    "SelfPlayLeagueBasedCallbackOldAPIStack",
+]

rllib/examples/multi_agent/utils/self_play_callback.py

@@ -0,0 +1,97 @@
+from collections import defaultdict
+
+import numpy as np
+
+from ray.rllib.callbacks.callbacks import RLlibCallback
+from ray.rllib.core.rl_module.rl_module import RLModuleSpec
+from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS
+
+
+class SelfPlayCallback(RLlibCallback):
+    def __init__(self, win_rate_threshold):
+        super().__init__()
+        # 0=RandomPolicy, 1=1st main policy snapshot,
+        # 2=2nd main policy snapshot, etc..
+        self.current_opponent = 0
+
+        self.win_rate_threshold = win_rate_threshold
+
+        # Report the matchup counters (who played against whom?).
+        self._matching_stats = defaultdict(int)
+
+    def on_episode_end(
+        self,
+        *,
+        episode,
+        env_runner,
+        metrics_logger,
+        env,
+        env_index,
+        rl_module,
+        **kwargs,
+    ) -> None:
+        # Compute the win rate for this episode and log it with a window of 100.
+        main_agent = 0 if episode.module_for(0) == "main" else 1
+        rewards = episode.get_rewards()
+        if main_agent in rewards:
+            main_won = rewards[main_agent][-1] == 1.0
+            metrics_logger.log_value(
+                "win_rate",
+                main_won,
+                reduce="mean",
+                window=100,
+            )
+
+    def on_train_result(self, *, algorithm, metrics_logger=None, result, **kwargs):
+        win_rate = result[ENV_RUNNER_RESULTS]["win_rate"]
+        print(f"Iter={algorithm.iteration} win-rate={win_rate} -> ", end="")
+        # If win rate is good -> Snapshot current policy and play against
+        # it next, keeping the snapshot fixed and only improving the "main"
+        # policy.
+        if win_rate > self.win_rate_threshold:
+            self.current_opponent += 1
+            new_module_id = f"main_v{self.current_opponent}"
+            print(f"adding new opponent to the mix ({new_module_id}).")
+
+            # Re-define the mapping function, such that "main" is forced
+            # to play against any of the previously played modules
+            # (excluding "random").
+            def agent_to_module_mapping_fn(agent_id, episode, **kwargs):
+                # agent_id = [0|1] -> policy depends on episode ID
+                # This way, we make sure that both modules sometimes play
+                # (start player) and sometimes agent1 (player to move 2nd).
+                opponent = "main_v{}".format(
+                    np.random.choice(list(range(1, self.current_opponent + 1)))
+                )
+                if hash(episode.id_) % 2 == agent_id:
+                    self._matching_stats[("main", opponent)] += 1
+                    return "main"
+                else:
+                    return opponent
+
+            main_module = algorithm.get_module("main")
+            algorithm.add_module(
+                module_id=new_module_id,
+                module_spec=RLModuleSpec.from_module(main_module),
+                new_agent_to_module_mapping_fn=agent_to_module_mapping_fn,
+            )
+            # TODO (sven): Maybe we should move this convenience step back into
+            #  `Algorithm.add_module()`? Would be less explicit, but also easier.
+            algorithm.set_state(
+                {
+                    "learner_group": {
+                        "learner": {
+                            "rl_module": {
+                                new_module_id: main_module.get_state(),
+                            }
+                        }
+                    }
+                }
+            )
+        else:
+            print("not good enough; will keep learning ...")
+
+        # +2 = main + random
+        result["league_size"] = self.current_opponent + 2
+
+        print(f"Matchups:\n{self._matching_stats}")

rllib/examples/multi_agent/utils/self_play_callback_old_api_stack.py

@@ -0,0 +1,78 @@
+import numpy as np
+
+from ray._common.deprecation import Deprecated
+from ray.rllib.callbacks.callbacks import RLlibCallback
+from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS
+
+
+@Deprecated(help="Use the example for the new RLlib API stack.", error=False)
+class SelfPlayCallbackOldAPIStack(RLlibCallback):
+    def __init__(self, win_rate_threshold):
+        super().__init__()
+        # 0=RandomPolicy, 1=1st main policy snapshot,
+        # 2=2nd main policy snapshot, etc..
+        self.current_opponent = 0
+
+        self.win_rate_threshold = win_rate_threshold
+
+    def on_train_result(self, *, algorithm, result, **kwargs):
+        # Get the win rate for the train batch.
+        # Note that normally, you should set up a proper evaluation config,
+        # such that evaluation always happens on the already updated policy,
+        # instead of on the already used train_batch.
+        main_rew = result[ENV_RUNNER_RESULTS]["hist_stats"].pop("policy_main_reward")
+        opponent_rew = list(result[ENV_RUNNER_RESULTS]["hist_stats"].values())[0]
+        assert len(main_rew) == len(opponent_rew)
+        won = 0
+        for r_main, r_opponent in zip(main_rew, opponent_rew):
+            if r_main > r_opponent:
+                won += 1
+        win_rate = won / len(main_rew)
+        result["win_rate"] = win_rate
+        print(f"Iter={algorithm.iteration} win-rate={win_rate} -> ", end="")
+        # If win rate is good -> Snapshot current policy and play against
+        # it next, keeping the snapshot fixed and only improving the "main"
+        # policy.
+        if win_rate > self.win_rate_threshold:
+            self.current_opponent += 1
+            new_pol_id = f"main_v{self.current_opponent}"
+            print(f"adding new opponent to the mix ({new_pol_id}).")
+
+            # Re-define the mapping function, such that "main" is forced
+            # to play against any of the previously played policies
+            # (excluding "random").
+            def policy_mapping_fn(agent_id, episode, worker, **kwargs):
+                # agent_id = [0|1] -> policy depends on episode ID
+                # This way, we make sure that both policies sometimes play
+                # (start player) and sometimes agent1 (player to move 2nd).
+                return (
+                    "main"
+                    if episode.episode_id % 2 == agent_id
+                    else "main_v{}".format(
+                        np.random.choice(list(range(1, self.current_opponent + 1)))
+                    )
+                )
+
+            main_policy = algorithm.get_policy("main")
+            new_policy = algorithm.add_policy(
+                policy_id=new_pol_id,
+                policy_cls=type(main_policy),
+                policy_mapping_fn=policy_mapping_fn,
+                config=main_policy.config,
+                observation_space=main_policy.observation_space,
+                action_space=main_policy.action_space,
+            )
+
+            # Set the weights of the new policy to the main policy.
+            # We'll keep training the main policy, whereas `new_pol_id` will
+            # remain fixed.
+            main_state = main_policy.get_state()
+            new_policy.set_state(main_state)
+            # We need to sync the just copied local weights (from main policy)
+            # to all the remote workers as well.
+            algorithm.env_runner_group.sync_weights()
+        else:
+            print("not good enough; will keep learning ...")
+
+        # +2 = main + random
+        result["league_size"] = self.current_opponent + 2

rllib/examples/multi_agent/utils/self_play_league_based_callback.py

@@ -0,0 +1,299 @@
+import re
+from collections import defaultdict
+from pprint import pprint
+
+import numpy as np
+
+from ray.rllib.callbacks.callbacks import RLlibCallback
+from ray.rllib.core.rl_module.rl_module import RLModuleSpec
+from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS
+
+
+class SelfPlayLeagueBasedCallback(RLlibCallback):
+    def __init__(self, win_rate_threshold):
+        super().__init__()
+        # All policies in the league.
+        self.main_policies = {"main", "main_0"}
+        self.main_exploiters = {"main_exploiter_0", "main_exploiter_1"}
+        self.league_exploiters = {"league_exploiter_0", "league_exploiter_1"}
+        # Set of currently trainable policies in the league.
+        self.trainable_policies = {"main"}
+        # Set of currently non-trainable (frozen) policies in the league.
+        self.non_trainable_policies = {
+            "main_0",
+            "league_exploiter_0",
+            "main_exploiter_0",
+        }
+        # The win-rate value reaching of which leads to a new module being added
+        # to the leage (frozen copy of main).
+        self.win_rate_threshold = win_rate_threshold
+        # Store the win rates for league overview printouts.
+        self.win_rates = {}
+
+        # Report the matchup counters (who played against whom?).
+        self._matching_stats = defaultdict(int)
+
+    def on_episode_end(
+        self,
+        *,
+        episode,
+        env_runner,
+        metrics_logger,
+        env,
+        env_index,
+        rl_module,
+        **kwargs,
+    ) -> None:
+
+        num_learning_policies = (
+            episode.module_for(0) in env_runner.config.policies_to_train
+        ) + (episode.module_for(1) in env_runner.config.policies_to_train)
+        # Make sure the mapping function doesn't match two non-trainables together.
+        # This would be a waste of EnvRunner resources.
+        # assert num_learning_policies > 0
+        # Ignore matches between two learning policies and don't count win-rates for
+        # these.
+        assert num_learning_policies > 0, (
+            f"agent=0 -> mod={episode.module_for(0)}; "
+            f"agent=1 -> mod={episode.module_for(1)}; "
+            f"EnvRunner.config.policies_to_train={env_runner.config.policies_to_train}"
+        )
+        if num_learning_policies == 1:
+            # Compute the win rate for this episode (only looking at non-trained
+            # opponents, such as random or frozen policies) and log it with some window.
+            rewards_dict = episode.get_rewards()
+            for aid, rewards in rewards_dict.items():
+                mid = episode.module_for(aid)
+                won = rewards[-1] == 1.0
+                metrics_logger.log_value(
+                    f"win_rate_{mid}",
+                    won,
+                    window=100,
+                )
+
+    def on_train_result(self, *, algorithm, metrics_logger=None, result, **kwargs):
+        local_worker = algorithm.env_runner
+
+        # Avoid `self` being pickled into the remote function below.
+        _trainable_policies = self.trainable_policies
+
+        # Get the win rate for the train batch.
+        # Note that normally, one should set up a proper evaluation config,
+        # such that evaluation always happens on the already updated policy,
+        # instead of on the already used train_batch.
+        league_changed = False
+        keys = [
+            k for k in result[ENV_RUNNER_RESULTS].keys() if k.startswith("win_rate_")
+        ]
+        for key in keys:
+            module_id = key[9:]
+            self.win_rates[module_id] = result[ENV_RUNNER_RESULTS][key]
+
+            # Policy is frozen; ignore.
+            if module_id in self.non_trainable_policies:
+                continue
+
+            print(
+                f"Iter={algorithm.iteration} {module_id}'s "
+                f"win-rate={self.win_rates[module_id]} -> ",
+                end="",
+            )
+
+            # If win rate is good -> Snapshot current policy and decide,
+            # whether to freeze the copy or not.
+            if self.win_rates[module_id] > self.win_rate_threshold:
+                is_main = re.match("^main(_\\d+)?$", module_id)
+                initializing_exploiters = False
+
+                # First time, main manages a decent win-rate against random:
+                # Add league_exploiter_1 and main_exploiter_1 as trainables to the mix.
+                if is_main and len(self.trainable_policies) == 1:
+                    initializing_exploiters = True
+                    self.trainable_policies.add("league_exploiter_1")
+                    self.trainable_policies.add("main_exploiter_1")
+                # If main manages to win (above threshold) against the entire league
+                # -> increase the league by another frozen copy of main,
+                # main-exploiters or league-exploiters.
+                else:
+                    keep_training = (
+                        False
+                        if is_main
+                        else np.random.choice([True, False], p=[0.3, 0.7])
+                    )
+                    if module_id in self.main_policies:
+                        new_mod_id = re.sub(
+                            "(main)(_\\d+)?$",
+                            f"\\1_{len(self.main_policies) - 1}",
+                            module_id,
+                        )
+                        self.main_policies.add(new_mod_id)
+                    elif module_id in self.main_exploiters:
+                        new_mod_id = re.sub(
+                            "_\\d+$", f"_{len(self.main_exploiters)}", module_id
+                        )
+                        self.main_exploiters.add(new_mod_id)
+                    else:
+                        new_mod_id = re.sub(
+                            "_\\d+$", f"_{len(self.league_exploiters)}", module_id
+                        )
+                        self.league_exploiters.add(new_mod_id)
+
+                    if keep_training:
+                        self.trainable_policies.add(new_mod_id)
+                    else:
+                        self.non_trainable_policies.add(new_mod_id)
+
+                    print(f"adding new opponents to the mix ({new_mod_id}).")
+
+                # Initialize state variablers for agent-to-module mapping. Note, we
+                # need to keep track of the league-exploiter to always match a
+                # non-trainable policy with a trainable one - otherwise matches are
+                # a waste of resources.
+                self.type_count = 0
+                self.exploiter = None
+
+                def agent_to_module_mapping_fn(agent_id, episode, **kwargs):
+                    # Pick whether this is ...
+                    type_ = np.random.choice([1, 2])
+
+                    # Each second third call reset state variables. Note, there will
+                    # be always two agents playing against each others.
+                    if self.type_count >= 2:
+                        # Reset the counter.
+                        self.type_count = 0
+                        # Set the exploiter to `None`.
+                        self.exploiter = None
+
+                    # Increment the counter for each agent.
+                    self.type_count += 1
+
+                    # 1) League exploiter vs any other.
+                    if type_ == 1:
+                        # Note, the exploiter could be either of `type_==1` or `type_==2`.
+                        if not self.exploiter:
+                            self.exploiter = "league_exploiter_" + str(
+                                np.random.choice(
+                                    list(range(len(self.league_exploiters)))
+                                )
+                            )
+                        # This league exploiter is frozen: Play against a
+                        # trainable policy.
+                        if self.exploiter not in self.trainable_policies:
+                            opponent = np.random.choice(list(self.trainable_policies))
+                        # League exploiter is trainable: Play against any other
+                        # non-trainable policy.
+                        else:
+                            opponent = np.random.choice(
+                                list(self.non_trainable_policies)
+                            )
+
+                        # Only record match stats once per match.
+                        if hash(episode.id_) % 2 == agent_id:
+                            self._matching_stats[(self.exploiter, opponent)] += 1
+                            return self.exploiter
+                        else:
+                            return opponent
+
+                    # 2) Main exploiter vs main.
+                    else:
+                        # Note, the exploiter could be either of `type_==1` or `type_==2`.
+                        if not self.exploiter:
+                            self.exploiter = "main_exploiter_" + str(
+                                np.random.choice(list(range(len(self.main_exploiters))))
+                            )
+                        # Main exploiter is frozen: Play against the main
+                        # policy.
+                        if self.exploiter not in self.trainable_policies:
+                            main = "main"
+                        # Main exploiter is trainable: Play against any
+                        # frozen main.
+                        else:
+                            main = np.random.choice(list(self.main_policies - {"main"}))
+
+                        # Only record match stats once per match.
+                        if hash(episode.id_) % 2 == agent_id:
+                            self._matching_stats[(self.exploiter, main)] += 1
+                            return self.exploiter
+                        else:
+                            return main
+
+                multi_rl_module = local_worker.module
+                main_module = multi_rl_module["main"]
+
+                # Set the weights of the new polic(y/ies).
+                if initializing_exploiters:
+                    main_state = main_module.get_state()
+                    multi_rl_module["main_0"].set_state(main_state)
+                    multi_rl_module["league_exploiter_1"].set_state(main_state)
+                    multi_rl_module["main_exploiter_1"].set_state(main_state)
+                    # We need to sync the just copied local weights to all the
+                    # remote workers and remote Learner workers as well.
+                    algorithm.env_runner_group.sync_weights(
+                        policies=["main_0", "league_exploiter_1", "main_exploiter_1"]
+                    )
+                    algorithm.learner_group.set_weights(multi_rl_module.get_state())
+                else:
+                    algorithm.add_module(
+                        module_id=new_mod_id,
+                        module_spec=RLModuleSpec.from_module(main_module),
+                    )
+                    # TODO (sven): Maybe we should move this convenience step back into
+                    #  `Algorithm.add_module()`? Would be less explicit, but also
+                    #  easier.
+                    algorithm.set_state(
+                        {
+                            "learner_group": {
+                                "learner": {
+                                    "rl_module": {
+                                        new_mod_id: multi_rl_module[
+                                            module_id
+                                        ].get_state(),
+                                    }
+                                }
+                            }
+                        }
+                    )
+
+                algorithm.env_runner_group.foreach_env_runner(
+                    lambda env_runner: env_runner.config.multi_agent(
+                        policy_mapping_fn=agent_to_module_mapping_fn,
+                        # This setting doesn't really matter for EnvRunners (no
+                        # training going on there, but we'll update this as well
+                        # here for good measure).
+                        policies_to_train=_trainable_policies,
+                    ),
+                    local_env_runner=True,
+                )
+                # Set all Learner workers' should_module_be_updated to the new
+                # value.
+                algorithm.learner_group.foreach_learner(
+                    func=lambda learner: learner.config.multi_agent(
+                        policies_to_train=_trainable_policies,
+                    ),
+                    timeout_seconds=0.0,  # fire-and-forget
+                )
+                league_changed = True
+            else:
+                print("not good enough; will keep learning ...")
+
+        # Add current league size to results dict.
+        result["league_size"] = len(self.non_trainable_policies) + len(
+            self.trainable_policies
+        )
+
+        if league_changed:
+            self._print_league()
+
+    def _print_league(self):
+        print("--- League ---")
+        print("Matchups:")
+        pprint(self._matching_stats)
+        print("Trainable policies (win-rates):")
+        for p in sorted(self.trainable_policies):
+            wr = self.win_rates[p] if p in self.win_rates else 0.0
+            print(f"\t{p}: {wr}")
+        print("Frozen policies:")
+        for p in sorted(self.non_trainable_policies):
+            wr = self.win_rates[p] if p in self.win_rates else 0.0
+            print(f"\t{p}: {wr}")
+        print()

rllib/examples/multi_agent/utils/self_play_league_based_callback_old_api_stack.py

@@ -0,0 +1,201 @@
+import re
+
+import numpy as np
+
+from ray._common.deprecation import Deprecated
+from ray.rllib.callbacks.callbacks import RLlibCallback
+from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS
+
+
+@Deprecated(help="Use the example for the new RLlib API stack", error=False)
+class SelfPlayLeagueBasedCallbackOldAPIStack(RLlibCallback):
+    def __init__(self, win_rate_threshold):
+        super().__init__()
+        # All policies in the league.
+        self.main_policies = {"main", "main_0"}
+        self.main_exploiters = {"main_exploiter_0", "main_exploiter_1"}
+        self.league_exploiters = {"league_exploiter_0", "league_exploiter_1"}
+        # Set of currently trainable policies in the league.
+        self.trainable_policies = {"main"}
+        # Set of currently non-trainable (frozen) policies in the league.
+        self.non_trainable_policies = {
+            "main_0",
+            "league_exploiter_0",
+            "main_exploiter_0",
+        }
+        # The win-rate value reaching of which leads to a new module being added
+        # to the leage (frozen copy of main).
+        self.win_rate_threshold = win_rate_threshold
+        # Store the win rates for league overview printouts.
+        self.win_rates = {}
+
+    def on_train_result(self, *, algorithm, result, **kwargs):
+        # Avoid `self` being pickled into the remote function below.
+        _trainable_policies = self.trainable_policies
+
+        # Get the win rate for the train batch.
+        # Note that normally, you should set up a proper evaluation config,
+        # such that evaluation always happens on the already updated policy,
+        # instead of on the already used train_batch.
+        for policy_id, rew in result[ENV_RUNNER_RESULTS]["hist_stats"].items():
+            mo = re.match("^policy_(.+)_reward$", policy_id)
+            if mo is None:
+                continue
+            policy_id = mo.group(1)
+
+            # Calculate this policy's win rate.
+            won = 0
+            for r in rew:
+                if r > 0.0:  # win = 1.0; loss = -1.0
+                    won += 1
+            win_rate = won / len(rew)
+            self.win_rates[policy_id] = win_rate
+
+            # Policy is frozen; ignore.
+            if policy_id in self.non_trainable_policies:
+                continue
+
+            print(
+                f"Iter={algorithm.iteration} {policy_id}'s " f"win-rate={win_rate} -> ",
+                end="",
+            )
+
+            # If win rate is good -> Snapshot current policy and decide,
+            # whether to freeze the copy or not.
+            if win_rate > self.win_rate_threshold:
+                is_main = re.match("^main(_\\d+)?$", policy_id)
+                initializing_exploiters = False
+
+                # First time, main manages a decent win-rate against random:
+                # Add league_exploiter_0 and main_exploiter_0 to the mix.
+                if is_main and len(self.trainable_policies) == 1:
+                    initializing_exploiters = True
+                    self.trainable_policies.add("league_exploiter_0")
+                    self.trainable_policies.add("main_exploiter_0")
+                else:
+                    keep_training = (
+                        False
+                        if is_main
+                        else np.random.choice([True, False], p=[0.3, 0.7])
+                    )
+                    if policy_id in self.main_policies:
+                        new_pol_id = re.sub(
+                            "_\\d+$", f"_{len(self.main_policies) - 1}", policy_id
+                        )
+                        self.main_policies.add(new_pol_id)
+                    elif policy_id in self.main_exploiters:
+                        new_pol_id = re.sub(
+                            "_\\d+$", f"_{len(self.main_exploiters)}", policy_id
+                        )
+                        self.main_exploiters.add(new_pol_id)
+                    else:
+                        new_pol_id = re.sub(
+                            "_\\d+$", f"_{len(self.league_exploiters)}", policy_id
+                        )
+                        self.league_exploiters.add(new_pol_id)
+
+                    if keep_training:
+                        self.trainable_policies.add(new_pol_id)
+                    else:
+                        self.non_trainable_policies.add(new_pol_id)
+
+                    print(f"adding new opponents to the mix ({new_pol_id}).")
+
+                # Update our mapping function accordingly.
+                def policy_mapping_fn(agent_id, episode, worker=None, **kwargs):
+                    # Pick, whether this is ...
+                    type_ = np.random.choice([1, 2])
+
+                    # 1) League exploiter vs any other.
+                    if type_ == 1:
+                        league_exploiter = "league_exploiter_" + str(
+                            np.random.choice(list(range(len(self.league_exploiters))))
+                        )
+                        # This league exploiter is frozen: Play against a
+                        # trainable policy.
+                        if league_exploiter not in self.trainable_policies:
+                            opponent = np.random.choice(list(self.trainable_policies))
+                        # League exploiter is trainable: Play against any other
+                        # non-trainable policy.
+                        else:
+                            opponent = np.random.choice(
+                                list(self.non_trainable_policies)
+                            )
+                        print(f"{league_exploiter} vs {opponent}")
+                        return (
+                            league_exploiter
+                            if episode.episode_id % 2 == agent_id
+                            else opponent
+                        )
+
+                    # 2) Main exploiter vs main.
+                    else:
+                        main_exploiter = "main_exploiter_" + str(
+                            np.random.choice(list(range(len(self.main_exploiters))))
+                        )
+                        # Main exploiter is frozen: Play against the main
+                        # policy.
+                        if main_exploiter not in self.trainable_policies:
+                            main = "main"
+                        # Main exploiter is trainable: Play against any
+                        # frozen main.
+                        else:
+                            main = np.random.choice(list(self.main_policies - {"main"}))
+                        # print(f"{main_exploiter} vs {main}")
+                        return (
+                            main_exploiter
+                            if episode.episode_id % 2 == agent_id
+                            else main
+                        )
+
+                # Set the weights of the new polic(y/ies).
+                if initializing_exploiters:
+                    main_state = algorithm.get_policy("main").get_state()
+                    pol_map = algorithm.env_runner.policy_map
+                    pol_map["main_0"].set_state(main_state)
+                    pol_map["league_exploiter_1"].set_state(main_state)
+                    pol_map["main_exploiter_1"].set_state(main_state)
+                    # We need to sync the just copied local weights to all the
+                    # remote workers as well.
+                    algorithm.env_runner_group.sync_weights(
+                        policies=["main_0", "league_exploiter_1", "main_exploiter_1"]
+                    )
+
+                    def _set(worker):
+                        worker.set_policy_mapping_fn(policy_mapping_fn)
+                        worker.set_is_policy_to_train(_trainable_policies)
+
+                    algorithm.env_runner_group.foreach_env_runner(_set)
+                else:
+                    base_pol = algorithm.get_policy(policy_id)
+                    new_policy = algorithm.add_policy(
+                        policy_id=new_pol_id,
+                        policy_cls=type(base_pol),
+                        policy_mapping_fn=policy_mapping_fn,
+                        policies_to_train=self.trainable_policies,
+                        config=base_pol.config,
+                        observation_space=base_pol.observation_space,
+                        action_space=base_pol.action_space,
+                    )
+                    main_state = base_pol.get_state()
+                    new_policy.set_state(main_state)
+                    # We need to sync the just copied local weights to all the
+                    # remote workers as well.
+                    algorithm.env_runner_group.sync_weights(policies=[new_pol_id])
+
+                self._print_league()
+
+            else:
+                print("not good enough; will keep learning ...")
+
+    def _print_league(self):
+        print("--- League ---")
+        print("Trainable policies (win-rates):")
+        for p in sorted(self.trainable_policies):
+            wr = self.win_rates[p] if p in self.win_rates else 0.0
+            print(f"\t{p}: {wr}")
+        print("Frozen policies:")
+        for p in sorted(self.non_trainable_policies):
+            wr = self.win_rates[p] if p in self.win_rates else 0.0
+            print(f"\t{p}: {wr}")
+        print()