wehub/ray-project--ray
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:17:40 +08:00
commit f1825c8ceb
10096 changed files with 2364182 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rllib/examples/multi_agent/__init__.py
View File
rllib/examples/multi_agent/custom_heuristic_policy.py
+99
View File
@@ -0,0 +1,99 @@
"""Example of running a custom heuristic (hand-coded) policy alongside trainable ones.
This example has two RLModules (as action computing policies):
(1) one trained by a PPOLearner
(2) one hand-coded policy that acts at random in the env (doesn't learn).
The environment is MultiAgentCartPole, in which there are n agents both policies
How to run this script
----------------------
`python [script file name].py --num-agents=2`
For debugging, use the following additional command line options
`--no-tune --num-env-runners=0`
which should allow you to set breakpoints anywhere in the RLlib code and
have the execution stop there for inspection and debugging.
For logging to your WandB account, use:
`--wandb-key=[your WandB API key] --wandb-project=[some project name]
--wandb-run-name=[optional: WandB run name (within the defined project)]`
Results to expect
-----------------
In the console output, you can see the PPO policy ("learnable_policy") does much
better than "random":
+-------------------+------------+----------+------+----------------+
| Trial name | status | loc | iter | total time (s) |
| | | | | |
|-------------------+------------+----------+------+----------------+
| PPO_multi_agen... | TERMINATED | 127. ... | 20 | 58.646 |
+-------------------+------------+----------+------+----------------+
+--------+-------------------+-----------------+--------------------+
| ts | combined reward | reward random | reward |
| | | | learnable_policy |
+--------+-------------------+-----------------+--------------------|
| 80000 | 481.26 | 78.41 | 464.41 |
+--------+-------------------+-----------------+--------------------+
"""
from ray.rllib.algorithms.ppo import PPOConfig
from ray.rllib.core.rl_module.multi_rl_module import MultiRLModuleSpec
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.examples.envs.classes.multi_agent import MultiAgentCartPole
from ray.rllib.examples.rl_modules.classes.random_rlm import RandomRLModule
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.tune.registry import register_env
parser = add_rllib_example_script_args(
default_iters=40, default_reward=500.0, default_timesteps=200000
)
parser.set_defaults(
num_agents=2,
)
if __name__ == "__main__":
args = parser.parse_args()
assert args.num_agents == 2, "Must set --num-agents=2 when running this script!"
# Simple environment with n independent cartpole entities.
register_env(
"multi_agent_cartpole",
lambda _: MultiAgentCartPole({"num_agents": args.num_agents}),
)
base_config = (
PPOConfig()
.environment("multi_agent_cartpole")
.multi_agent(
policies={"learnable_policy", "random"},
# Map to either random behavior or PPO learning behavior based on
# the agent's ID.
policy_mapping_fn=lambda agent_id, *args, **kwargs: [
"learnable_policy",
"random",
][agent_id % 2],
# We need to specify this here, b/c the `forward_train` method of
# `RandomRLModule` (ModuleID="random") throws a not-implemented error.
policies_to_train=["learnable_policy"],
)
.rl_module(
rl_module_spec=MultiRLModuleSpec(
rl_module_specs={
"learnable_policy": RLModuleSpec(),
"random": RLModuleSpec(module_class=RandomRLModule),
}
),
)
)
run_rllib_example_script_experiment(base_config, args)
rllib/examples/multi_agent/different_spaces_for_agents.py
+108
View File
@@ -0,0 +1,108 @@
"""
Example showing how to create a multi-agent env, in which the different agents
have different observation and action spaces.
These spaces do NOT necessarily have to be specified manually by the user. Instead,
RLlib tries to automatically infer them from the env provided spaces dicts
(agentID -> obs/act space) and the policy mapping fn (mapping agent IDs to policy IDs).
How to run this script
----------------------
`python [script file name].py --num-agents=2`
For debugging, use the following additional command line options
`--no-tune --num-env-runners=0`
which should allow you to set breakpoints anywhere in the RLlib code and
have the execution stop there for inspection and debugging.
For logging to your WandB account, use:
`--wandb-key=[your WandB API key] --wandb-project=[some project name]
--wandb-run-name=[optional: WandB run name (within the defined project)]`
"""
import gymnasium as gym
from ray.rllib.env.multi_agent_env import MultiAgentEnv
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.tune.registry import get_trainable_cls
class BasicMultiAgentMultiSpaces(MultiAgentEnv):
"""A simple multi-agent example environment where agents have different spaces.
agent0: obs=Box(10,), act=Discrete(2)
agent1: obs=Box(20,), act=Discrete(3)
The logic of the env doesn't really matter for this example. The point of this env
is to show how to use multi-agent envs, in which the different agents utilize
different obs- and action spaces.
"""
def __init__(self, config=None):
self.agents = ["agent0", "agent1"]
self.terminateds = set()
self.truncateds = set()
# Provide full (preferred format) observation- and action-spaces as Dicts
# mapping agent IDs to the individual agents' spaces.
self.observation_spaces = {
"agent0": gym.spaces.Box(low=-1.0, high=1.0, shape=(10,)),
"agent1": gym.spaces.Box(low=-1.0, high=1.0, shape=(20,)),
}
self.action_spaces = {
"agent0": gym.spaces.Discrete(2),
"agent1": gym.spaces.Discrete(3),
}
super().__init__()
def reset(self, *, seed=None, options=None):
self.terminateds = set()
self.truncateds = set()
return {i: self.get_observation_space(i).sample() for i in self.agents}, {}
def step(self, action_dict):
obs, rew, terminated, truncated, info = {}, {}, {}, {}, {}
for i, action in action_dict.items():
obs[i] = self.get_observation_space(i).sample()
rew[i] = 0.0
terminated[i] = False
truncated[i] = False
info[i] = {}
terminated["__all__"] = len(self.terminateds) == len(self.agents)
truncated["__all__"] = len(self.truncateds) == len(self.agents)
return obs, rew, terminated, truncated, info
parser = add_rllib_example_script_args(
default_iters=10, default_reward=80.0, default_timesteps=10000
)
if __name__ == "__main__":
args = parser.parse_args()
base_config = (
get_trainable_cls(args.algo)
.get_default_config()
.environment(env=BasicMultiAgentMultiSpaces)
.training(train_batch_size=1024)
.multi_agent(
# Use a simple set of policy IDs. Spaces for the individual policies
# are inferred automatically using reverse lookup via the
# `policy_mapping_fn` and the env provided spaces for the different
# agents. Alternatively, you could use:
# policies: {main0: PolicySpec(...), main1: PolicySpec}
policies={"main0", "main1"},
# Simple mapping fn, mapping agent0 to main0 and agent1 to main1.
policy_mapping_fn=(lambda aid, episode, **kw: f"main{aid[-1]}"),
# Only train main0.
policies_to_train=["main0"],
)
)
run_rllib_example_script_experiment(base_config, args)
rllib/examples/multi_agent/multi_agent_cartpole.py
+66
View File
@@ -0,0 +1,66 @@
"""Simple example of setting up an agent-to-module mapping function.
How to run this script
----------------------
`python [script file name].py --num-agents=2`
Control the number of agents and policies (RLModules) via --num-agents and
--num-policies.
For debugging, use the following additional command line options
`--no-tune --num-env-runners=0`
which should allow you to set breakpoints anywhere in the RLlib code and
have the execution stop there for inspection and debugging.
For logging to your WandB account, use:
`--wandb-key=[your WandB API key] --wandb-project=[some project name]
--wandb-run-name=[optional: WandB run name (within the defined project)]`
"""
from ray.rllib.examples.envs.classes.multi_agent import MultiAgentCartPole
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.tune.registry import get_trainable_cls, register_env
parser = add_rllib_example_script_args(
default_iters=200,
default_timesteps=100000,
default_reward=600.0,
)
# TODO (sven): This arg is currently ignored (hard-set to 2).
parser.add_argument(
"--num-policies",
type=int,
default=2,
)
if __name__ == "__main__":
args = parser.parse_args()
# Register our environment with tune.
if args.num_agents > 0:
register_env(
"env",
lambda _: MultiAgentCartPole(config={"num_agents": args.num_agents}),
)
base_config = (
get_trainable_cls(args.algo)
.get_default_config()
.environment("env" if args.num_agents > 0 else "CartPole-v1")
.env_runners(
num_envs_per_env_runner=20,
)
)
# Add a simple multi-agent setup.
if args.num_agents > 0:
base_config.multi_agent(
policies={f"p{i}" for i in range(args.num_agents)},
policy_mapping_fn=lambda aid, *a, **kw: f"p{aid}",
)
run_rllib_example_script_experiment(base_config, args)
rllib/examples/multi_agent/multi_agent_pendulum.py
+77
View File
@@ -0,0 +1,77 @@
"""Simple example of setting up an agent-to-module mapping function.
How to run this script
----------------------
`python [script file name].py --num-agents=2`
Control the number of agents and policies (RLModules) via --num-agents and
--num-policies.
For debugging, use the following additional command line options
`--no-tune --num-env-runners=0`
which should allow you to set breakpoints anywhere in the RLlib code and
have the execution stop there for inspection and debugging.
For logging to your WandB account, use:
`--wandb-key=[your WandB API key] --wandb-project=[some project name]
--wandb-run-name=[optional: WandB run name (within the defined project)]`
"""
from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
from ray.rllib.examples.envs.classes.multi_agent import MultiAgentPendulum
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.tune.registry import get_trainable_cls, register_env
parser = add_rllib_example_script_args(
default_iters=200,
default_timesteps=100000,
default_reward=-400.0,
)
# TODO (sven): This arg is currently ignored (hard-set to 2).
parser.add_argument(
"--num-policies",
type=int,
default=2,
)
if __name__ == "__main__":
args = parser.parse_args()
# Register our environment with tune.
if args.num_agents > 0:
register_env(
"env",
lambda _: MultiAgentPendulum(config={"num_agents": args.num_agents}),
)
base_config = (
get_trainable_cls(args.algo)
.get_default_config()
.environment("env" if args.num_agents > 0 else "Pendulum-v1")
.training(
train_batch_size_per_learner=512,
minibatch_size=64,
lambda_=0.1,
gamma=0.95,
lr=0.0003,
model={"fcnet_activation": "relu"},
vf_clip_param=10.0,
)
.rl_module(
model_config=DefaultModelConfig(fcnet_activation="relu"),
)
)
# Add a simple multi-agent setup.
if args.num_agents > 0:
base_config.multi_agent(
policies={f"p{i}" for i in range(args.num_agents)},
policy_mapping_fn=lambda aid, *a, **kw: f"p{aid}",
)
# Augment
run_rllib_example_script_experiment(base_config, args)
rllib/examples/multi_agent/pettingzoo_independent_learning.py
+104
View File
@@ -0,0 +1,104 @@
"""Runs the PettingZoo Waterworld env in RLlib using independent multi-agent learning.
See: https://pettingzoo.farama.org/environments/sisl/waterworld/
for more details on the environment.
How to run this script
----------------------
`python [script file name].py --num-agents=2`
Control the number of agents and policies (RLModules) via --num-agents and
--num-policies.
This works with hundreds of agents and policies, but note that initializing
many policies might take some time.
For debugging, use the following additional command line options
`--no-tune --num-env-runners=0`
which should allow you to set breakpoints anywhere in the RLlib code and
have the execution stop there for inspection and debugging.
For logging to your WandB account, use:
`--wandb-key=[your WandB API key] --wandb-project=[some project name]
--wandb-run-name=[optional: WandB run name (within the defined project)]`
Results to expect
-----------------
The above options can reach a combined reward of 0.0 or more after about 500k env
timesteps. Keep in mind, though, that due to the separate value functions (and
learned policies in general), one agent's gain (in per-agent reward) might cause the
other agent's reward to decrease at the same time. However, over time, both agents
should simply improve.
+---------------------+------------+-----------------+--------+------------------+
| Trial name | status | loc | iter | total time (s) |
|---------------------+------------+-----------------+--------+------------------+
| PPO_env_a82fc_00000 | TERMINATED | 127.0.0.1:28346 | 124 | 363.599 |
+---------------------+------------+-----------------+--------+------------------+
+--------+-------------------+--------------------+--------------------+
| ts | combined reward | reward pursuer_1 | reward pursuer_0 |
+--------+-------------------+--------------------+--------------------|
| 496000 | 2.24542 | -34.6869 | 36.9324 |
+--------+-------------------+--------------------+--------------------+
Note that the two agents (`pursuer_0` and `pursuer_1`) are optimized on the exact same
objective and thus differences in the rewards can be attributed to weight initialization
(and sampling randomness) only.
"""
from pettingzoo.sisl import waterworld_v4
from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
from ray.rllib.core.rl_module.multi_rl_module import MultiRLModuleSpec
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.env.wrappers.pettingzoo_env import PettingZooEnv
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.tune.registry import get_trainable_cls, register_env
parser = add_rllib_example_script_args(
default_iters=200,
default_timesteps=1000000,
default_reward=0.0,
)
if __name__ == "__main__":
args = parser.parse_args()
assert args.num_agents > 0, "Must set --num-agents > 0 when running this script!"
# Here, we use the "Agent Environment Cycle" (AEC) PettingZoo environment type.
# For a "Parallel" environment example, see the rock paper scissors examples
# in this same repository folder.
register_env("env", lambda _: PettingZooEnv(waterworld_v4.env()))
# Policies are called just like the agents (exact 1:1 mapping).
policies = {f"pursuer_{i}" for i in range(args.num_agents)}
base_config = (
get_trainable_cls(args.algo)
.get_default_config()
.environment("env")
.multi_agent(
policies=policies,
# Exact 1:1 mapping from AgentID to ModuleID.
policy_mapping_fn=(lambda aid, *args, **kwargs: aid),
)
.training(
vf_loss_coeff=0.005,
)
.rl_module(
rl_module_spec=MultiRLModuleSpec(
rl_module_specs={p: RLModuleSpec() for p in policies},
),
model_config=DefaultModelConfig(vf_share_layers=True),
)
)
run_rllib_example_script_experiment(base_config, args)
rllib/examples/multi_agent/pettingzoo_parameter_sharing.py
+101
View File
@@ -0,0 +1,101 @@
"""Runs the PettingZoo Waterworld multi-agent env in RLlib using single policy learning.
Other than the `pettingzoo_independent_learning.py` example (in this same folder),
this example simply trains a single policy (shared by all agents).
See: https://pettingzoo.farama.org/environments/sisl/waterworld/
for more details on the environment.
How to run this script
----------------------
`python [script file name].py --num-agents=2`
Control the number of agents and policies (RLModules) via --num-agents and
--num-policies.
This works with hundreds of agents and policies, but note that initializing
many policies might take some time.
For debugging, use the following additional command line options
`--no-tune --num-env-runners=0`
which should allow you to set breakpoints anywhere in the RLlib code and
have the execution stop there for inspection and debugging.
For logging to your WandB account, use:
`--wandb-key=[your WandB API key] --wandb-project=[some project name]
--wandb-run-name=[optional: WandB run name (within the defined project)]`
Results to expect
-----------------
The above options can reach a combined reward of roughly ~0.0 after about 500k-1M env
timesteps. Keep in mind, though, that in this setup, the agents do not have the
opportunity to benefit from or even out other agents' mistakes (and behavior in general)
as everyone is using the same policy. Hence, this example learns a more generic policy,
which might be less specialized to certain "niche exploitation opportunities" inside
the env:
+---------------------+----------+-----------------+--------+-----------------+
| Trial name | status | loc | iter | total time (s) |
|---------------------+----------+-----------------+--------+-----------------+
| PPO_env_91f49_00000 | RUNNING | 127.0.0.1:63676 | 200 | 605.176 |
+---------------------+----------+-----------------+--------+-----------------+
+--------+-------------------+-------------+
| ts | combined reward | reward p0 |
+--------+-------------------+-------------|
| 800000 | 0.323752 | 0.161876 |
+--------+-------------------+-------------+
"""
from pettingzoo.sisl import waterworld_v4
from ray.rllib.core.rl_module.multi_rl_module import MultiRLModuleSpec
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.env.wrappers.pettingzoo_env import PettingZooEnv
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.tune.registry import get_trainable_cls, register_env
parser = add_rllib_example_script_args(
default_iters=200,
default_timesteps=1000000,
default_reward=0.0,
)
if __name__ == "__main__":
args = parser.parse_args()
assert args.num_agents > 0, "Must set --num-agents > 0 when running this script!"
# Here, we use the "Agent Environment Cycle" (AEC) PettingZoo environment type.
# For a "Parallel" environment example, see the rock paper scissors examples
# in this same repository folder.
register_env("env", lambda _: PettingZooEnv(waterworld_v4.env()))
base_config = (
get_trainable_cls(args.algo)
.get_default_config()
.environment("env")
.multi_agent(
policies={"p0"},
# All agents map to the exact same policy.
policy_mapping_fn=(lambda aid, *args, **kwargs: "p0"),
)
.training(
model={
"vf_share_layers": True,
},
vf_loss_coeff=0.005,
)
.rl_module(
rl_module_spec=MultiRLModuleSpec(
rl_module_specs={"p0": RLModuleSpec()},
),
)
)
run_rllib_example_script_experiment(base_config, args)
rllib/examples/multi_agent/rock_paper_scissors_heuristic_vs_learned.py
+152
View File
@@ -0,0 +1,152 @@
"""A simple multi-agent env with two agents play rock paper scissors.
This demonstrates running the following policies in competition:
Agent 1: heuristic policy of repeating the same move
OR: heuristic policy of beating the last opponent move
Agent 2: Simple, feedforward PPO policy
OR: PPO Policy with an LSTM network
How to run this script
----------------------
`python [script file name].py --num-agents=2 [--use-lstm]?`
Without `--use-lstm`, Agent 2 should quickly reach a reward of ~7.0, always
beating the `always_same` policy, but only 50% of the time beating the `beat_last`
policy.
With `--use-lstm`, Agent 2 should eventually(!) reach a reward of >9.0 (always
beating both the `always_same` policy and the `beat_last` policy).
For debugging, use the following additional command line options
`--no-tune --num-env-runners=0`
which should allow you to set breakpoints anywhere in the RLlib code and
have the execution stop there for inspection and debugging.
For logging to your WandB account, use:
`--wandb-key=[your WandB API key] --wandb-project=[some project name]
--wandb-run-name=[optional: WandB run name (within the defined project)]`
"""
import random
import gymnasium as gym
from pettingzoo.classic import rps_v2
from ray.rllib.connectors.env_to_module import FlattenObservations
from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
from ray.rllib.core.rl_module.multi_rl_module import MultiRLModuleSpec
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.env.wrappers.pettingzoo_env import ParallelPettingZooEnv
from ray.rllib.examples.rl_modules.classes import (
AlwaysSameHeuristicRLM,
BeatLastHeuristicRLM,
)
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.rllib.utils.metrics import (
ENV_RUNNER_RESULTS,
NUM_ENV_STEPS_SAMPLED_LIFETIME,
)
from ray.tune.registry import get_trainable_cls, register_env
from ray.tune.result import TRAINING_ITERATION
parser = add_rllib_example_script_args(
default_iters=50,
default_timesteps=200000,
default_reward=6.0,
)
parser.set_defaults(
num_agents=2,
)
parser.add_argument(
"--use-lstm",
action="store_true",
help="Whether to use an LSTM wrapped module instead of a simple MLP one. With LSTM "
"the reward diff can reach 7.0, without only 5.0.",
)
register_env(
"pettingzoo_rps",
lambda _: ParallelPettingZooEnv(rps_v2.parallel_env()),
)
if __name__ == "__main__":
args = parser.parse_args()
assert args.num_agents == 2, "Must set --num-agents=2 when running this script!"
base_config = (
get_trainable_cls(args.algo)
.get_default_config()
.environment("pettingzoo_rps")
.env_runners(
env_to_module_connector=lambda env, spaces, device: (
# `agent_ids=...`: Only flatten obs for the learning RLModule.
FlattenObservations(multi_agent=True, agent_ids={"player_0"}),
),
)
.multi_agent(
policies={"always_same", "beat_last", "learned"},
# Let learning Policy always play against either heuristic one:
# `always_same` or `beat_last`.
policy_mapping_fn=lambda aid, episode: (
"learned"
if aid == "player_0"
else random.choice(["always_same", "beat_last"])
),
# Must define this as both heuristic RLMs will throw an error, if their
# `forward_train` is called.
policies_to_train=["learned"],
)
.training(
vf_loss_coeff=0.005,
)
.rl_module(
rl_module_spec=MultiRLModuleSpec(
rl_module_specs={
"always_same": RLModuleSpec(
module_class=AlwaysSameHeuristicRLM,
observation_space=gym.spaces.Discrete(4),
action_space=gym.spaces.Discrete(3),
),
"beat_last": RLModuleSpec(
module_class=BeatLastHeuristicRLM,
observation_space=gym.spaces.Discrete(4),
action_space=gym.spaces.Discrete(3),
),
"learned": RLModuleSpec(
model_config=DefaultModelConfig(
use_lstm=args.use_lstm,
# Use a simpler FCNet when we also have an LSTM.
fcnet_hiddens=[32] if args.use_lstm else [256, 256],
lstm_cell_size=256,
max_seq_len=15,
vf_share_layers=True,
),
),
}
),
)
)
# Make `args.stop_reward` "point" to the reward of the learned policy.
stop = {
TRAINING_ITERATION: args.stop_iters,
f"{ENV_RUNNER_RESULTS}/module_episode_returns_mean/learned": args.stop_reward,
NUM_ENV_STEPS_SAMPLED_LIFETIME: args.stop_timesteps,
}
run_rllib_example_script_experiment(
base_config,
args,
stop=stop,
success_metric={
f"{ENV_RUNNER_RESULTS}/module_episode_returns_mean/learned": (
args.stop_reward
),
},
)
rllib/examples/multi_agent/rock_paper_scissors_learned_vs_learned.py
+91
View File
@@ -0,0 +1,91 @@
"""A simple multi-agent env with two agents play rock paper scissors.
This demonstrates running two learning policies in competition, both using the same
RLlib algorithm (PPO by default).
The combined reward as well as individual rewards should roughly remain at 0.0 as no
policy should - in the long run - be able to learn a better strategy than chosing
actions at random. However, it could be possible that - for some time - one or the other
policy can exploit a "stochastic weakness" of the opponent policy. For example a policy
`A` learns that its opponent `B` has learnt to choose "paper" more often, which in
return makes `A` choose "scissors" more often as a countermeasure.
"""
import re
from pettingzoo.classic import rps_v2
from ray.rllib.connectors.env_to_module import FlattenObservations
from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
from ray.rllib.core.rl_module.multi_rl_module import MultiRLModuleSpec
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.env.wrappers.pettingzoo_env import ParallelPettingZooEnv
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.tune.registry import get_trainable_cls, register_env
parser = add_rllib_example_script_args(
default_iters=50,
default_timesteps=200000,
default_reward=6.0,
)
parser.set_defaults(
num_agents=2,
)
parser.add_argument(
"--use-lstm",
action="store_true",
help="Whether to use an LSTM wrapped module instead of a simple MLP one. With LSTM "
"the reward diff can reach 7.0, without only 5.0.",
)
register_env(
"pettingzoo_rps",
lambda _: ParallelPettingZooEnv(rps_v2.parallel_env()),
)
if __name__ == "__main__":
args = parser.parse_args()
assert args.num_agents == 2, "Must set --num-agents=2 when running this script!"
base_config = (
get_trainable_cls(args.algo)
.get_default_config()
.environment("pettingzoo_rps")
.env_runners(
env_to_module_connector=(
lambda env, spaces, device: FlattenObservations(multi_agent=True)
),
)
.multi_agent(
policies={"p0", "p1"},
# `player_0` uses `p0`, `player_1` uses `p1`.
policy_mapping_fn=lambda aid, episode: re.sub("^player_", "p", aid),
)
.training(
vf_loss_coeff=0.005,
)
.rl_module(
model_config=DefaultModelConfig(
use_lstm=args.use_lstm,
# Use a simpler FCNet when we also have an LSTM.
fcnet_hiddens=[32] if args.use_lstm else [256, 256],
lstm_cell_size=256,
max_seq_len=15,
vf_share_layers=True,
),
rl_module_spec=MultiRLModuleSpec(
rl_module_specs={
"p0": RLModuleSpec(),
"p1": RLModuleSpec(),
}
),
)
)
run_rllib_example_script_experiment(base_config, args)
rllib/examples/multi_agent/self_play_footsies.py
+129
View File
@@ -0,0 +1,129 @@
"""
Multi-agent RLlib Footsies Simplified Example (PPO)
About:
- This example as a simplified version of "rllib/examples/ppo/multi_agent_footsies_ppo.py",
which has more detailed comments and instructions. Please refer to that example for more information.
- This example is created to test the self-play training progression with footsies.
- Simplified version runs with single learner (cpu), single env runner, and single eval env runner.
"""
import platform
from pathlib import Path
from ray.rllib.examples.algorithms.ppo.multi_agent_footsies_ppo import (
config,
env_creator,
stop,
)
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.tune.registry import register_env
parser = add_rllib_example_script_args(
default_iters=500,
default_timesteps=5_000_000,
)
parser.add_argument(
"--train-start-port",
type=int,
default=45001,
help="First port number for the Footsies training environment server (default: 45001). Each server gets its own port.",
)
parser.add_argument(
"--eval-start-port",
type=int,
default=55001,
help="First port number for the Footsies evaluation environment server (default: 55001) Each server gets its own port.",
)
parser.add_argument(
"--binary-download-dir",
type=Path,
default="/tmp/ray/binaries/footsies",
help="Directory to download Footsies binaries (default: /tmp/ray/binaries/footsies)",
)
parser.add_argument(
"--binary-extract-dir",
type=Path,
default="/tmp/ray/binaries/footsies",
help="Directory to extract Footsies binaries (default: /tmp/ray/binaries/footsies)",
)
parser.add_argument(
"--render",
action="store_true",
default=False,
help="Whether to render the Footsies environment. Default is False.",
)
parser.add_argument(
"--win-rate-threshold",
type=float,
default=0.55,
help="The main policy should have at least 'win-rate-threshold' win rate against the "
"other policy to advance to the next level. Moving to the next level "
"means adding a new policy to the mix.",
)
parser.add_argument(
"--target-mix-size",
type=int,
default=4,
help="Target number of policies (RLModules) in the mix to consider the test passed. "
"The initial mix size is 2: 'main policy' vs. 'other'. "
"`--target-mix-size=4` means that 2 new policies will be added to the mix. "
"Whether to add new policy is decided by checking the '--win-rate-threshold' condition. ",
)
parser.add_argument(
"--rollout-fragment-length",
type=int,
default=256,
help="The length of each rollout fragment to be collected by the EnvRunners when sampling.",
)
parser.add_argument(
"--log-unity-output",
action="store_true",
help="Whether to log Unity output (from the game engine). Default is False.",
default=False,
)
args = parser.parse_args()
register_env(name="FootsiesEnv", env_creator=env_creator)
stop["mix_size"] = args.target_mix_size
# Detect platform and choose appropriate binary
if platform.system() == "Darwin":
if args.render:
binary_to_download = "mac_windowed"
else:
binary_to_download = "mac_headless"
elif platform.system() == "Linux":
if args.render:
binary_to_download = "linux_windowed"
else:
binary_to_download = "linux_server"
else:
raise RuntimeError(f"Unsupported platform: {platform.system()}")
config.environment(
env="FootsiesEnv",
env_config={
"train_start_port": args.train_start_port,
"eval_start_port": args.eval_start_port,
"binary_download_dir": args.binary_download_dir,
"binary_extract_dir": args.binary_extract_dir,
"binary_to_download": binary_to_download,
"log_unity_output": args.log_unity_output,
},
).training(
train_batch_size_per_learner=args.rollout_fragment_length
* (args.num_env_runners or 1),
)
if __name__ == "__main__":
results = run_rllib_example_script_experiment(
base_config=config,
args=args,
stop=stop,
success_metric={"mix_size": args.target_mix_size},
)
rllib/examples/multi_agent/self_play_league_based_with_open_spiel.py
+284
View File
@@ -0,0 +1,284 @@
"""Example showing how to implement a league-based training workflow.
Uses the open spiel adapter of RLlib with the "markov_soccer" game and
a simplified multi-agent, league-based setup:
https://deepmind.com/blog/article/AlphaStar-Grandmaster-level-in- \
StarCraft-II-using-multi-agent-reinforcement-learning
Our league consists of three groups of policies:
- main policies: The current main policy plus prior versions of it.
- main exploiters: Trained by playing only against different "main policies".
- league exploiters: Trained by playing against any policy in the league.
We start with 1 policy from each group, setting all 3 of these to an initial
PPO policy and allowing all 3 policies to be trained.
After each train update - via our custom callback - we decide for each
trainable policy, whether to make a copy and freeze it. Frozen policies
will not be altered anymore. However, they remain in the league for
future matches against trainable policies.
Matchmaking happens via a policy_mapping_fn, which needs to be altered
after every change (addition) to the league. The mapping function
randomly maps agents in a way, such that:
- Frozen main exploiters play against the one (currently trainable) main
policy.
- Trainable main exploiters play against any main policy (including already
frozen main policies).
- Frozen league exploiters play against any trainable policy in the league.
- Trainable league exploiters play against any policy in the league.
After training for n iterations, a configurable number of episodes can
be played by the user against the "main" agent on the command line.
"""
import functools
import numpy as np
import torch
import ray
from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
from ray.rllib.core.rl_module.multi_rl_module import MultiRLModuleSpec
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.env.utils import try_import_open_spiel, try_import_pyspiel
from ray.rllib.env.wrappers.open_spiel import OpenSpielEnv
from ray.rllib.examples._old_api_stack.policy.random_policy import RandomPolicy
from ray.rllib.examples.multi_agent.utils import (
SelfPlayLeagueBasedCallback,
SelfPlayLeagueBasedCallbackOldAPIStack,
ask_user_for_action,
)
from ray.rllib.examples.rl_modules.classes.random_rlm import RandomRLModule
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.rllib.policy.policy import PolicySpec
from ray.rllib.utils.metrics import NUM_ENV_STEPS_SAMPLED_LIFETIME
from ray.tune.registry import get_trainable_cls, register_env
from ray.tune.result import TRAINING_ITERATION
open_spiel = try_import_open_spiel(error=True)
pyspiel = try_import_pyspiel(error=True)
# Import after try_import_open_spiel, so we can error out with hints
from open_spiel.python.rl_environment import Environment # noqa: E402
parser = add_rllib_example_script_args(default_timesteps=2000000)
parser.set_defaults(
env="markov_soccer",
num_env_runners=2,
checkpoint_freq=1,
checkpoint_at_end=True,
)
parser.add_argument(
"--win-rate-threshold",
type=float,
default=0.85,
help="Win-rate at which we setup another opponent by freezing the "
"current main policy and playing against a uniform distribution "
"of previously frozen 'main's from here on.",
)
parser.add_argument(
"--min-league-size",
type=float,
default=8,
help="Minimum number of policies/RLModules to consider the test passed. "
"The initial league size is 2: `main` and `random`. "
"`--min-league-size=3` thus means that one new policy/RLModule has been "
"added so far (b/c the `main` one has reached the `--win-rate-threshold "
"against the `random` Policy/RLModule).",
)
parser.add_argument(
"--num-episodes-human-play",
type=int,
default=0,
help="How many episodes to play against the user on the command "
"line after training has finished.",
)
parser.add_argument(
"--from-checkpoint",
type=str,
default=None,
help="Full path to a checkpoint file for restoring a previously saved "
"Algorithm state.",
)
if __name__ == "__main__":
args = parser.parse_args()
register_env(
"open_spiel_env",
lambda _: OpenSpielEnv(pyspiel.load_game(args.env)),
)
def policy_mapping_fn(agent_id, episode, worker=None, **kwargs):
# At first, only have main play against the random main exploiter.
return "main" if episode.episode_id % 2 == agent_id else "main_exploiter_0"
def agent_to_module_mapping_fn(agent_id, episode, **kwargs):
# At first, only have main play against the random main exploiter.
return "main" if hash(episode.id_) % 2 == agent_id else "main_exploiter_0"
def _get_multi_agent():
names = {
# Our main policy, we'd like to optimize.
"main",
# First frozen version of main (after we reach n% win-rate).
"main_0",
# Initial main exploiters (one random, one trainable).
"main_exploiter_0",
"main_exploiter_1",
# Initial league exploiters (one random, one trainable).
"league_exploiter_0",
"league_exploiter_1",
}
if not args.old_api_stack:
policies = names
spec = {
mid: RLModuleSpec(
module_class=(
RandomRLModule
if mid in ["main_exploiter_0", "league_exploiter_0"]
else None
),
model_config=DefaultModelConfig(
fcnet_hiddens=[1024, 1024],
# fcnet_activation="tanh",
),
)
for mid in names
}
else:
policies = {
mid: PolicySpec(
policy_class=(
RandomPolicy
if mid in ["main_exploiter_0", "league_exploiter_0"]
else None
)
)
for mid in names
}
spec = None
return {"policies": policies, "spec": spec}
config = (
get_trainable_cls(args.algo)
.get_default_config()
.environment("open_spiel_env")
# Set up the main piece in this experiment: The league-bases self-play
# callback, which controls adding new policies/Modules to the league and
# properly matching the different policies in the league with each other.
.callbacks(
functools.partial(
SelfPlayLeagueBasedCallback
if not args.old_api_stack
else SelfPlayLeagueBasedCallbackOldAPIStack,
win_rate_threshold=args.win_rate_threshold,
)
)
.env_runners(
num_envs_per_env_runner=1 if not args.old_api_stack else 5,
)
.training(
num_epochs=20,
)
.multi_agent(
# Initial policy map: All PPO. This will be expanded
# to more policy snapshots. This is done in the
# custom callback defined above (`LeagueBasedSelfPlayCallback`).
policies=_get_multi_agent()["policies"],
policy_mapping_fn=(
agent_to_module_mapping_fn
if not args.old_api_stack
else policy_mapping_fn
),
# At first, only train main_0 (until good enough to win against
# random).
policies_to_train=["main"],
)
.rl_module(
rl_module_spec=MultiRLModuleSpec(
rl_module_specs=_get_multi_agent()["spec"]
),
)
)
# Run everything as configured.
# Train the "main" policy to play really well using self-play.
results = None
if not args.from_checkpoint:
stop = {
NUM_ENV_STEPS_SAMPLED_LIFETIME: args.stop_timesteps,
TRAINING_ITERATION: args.stop_iters,
"league_size": args.min_league_size,
}
results = run_rllib_example_script_experiment(
config, args, stop=stop, keep_ray_up=True
)
# Restore trained Algorithm (set to non-explore behavior) and play against
# human on command line.
if args.num_episodes_human_play > 0:
num_episodes = 0
# Switch off exploration for better inference performance.
config.explore = False
algo = config.build()
if args.from_checkpoint:
algo.restore(args.from_checkpoint)
else:
checkpoint = results.get_best_result().checkpoint
if not checkpoint:
raise ValueError("No last checkpoint found in results!")
algo.restore(checkpoint)
if not args.old_api_stack:
rl_module = algo.get_module("main")
# Play from the command line against the trained agent
# in an actual (non-RLlib-wrapped) open-spiel env.
human_player = 1
env = Environment(args.env)
while num_episodes < args.num_episodes_human_play:
print("You play as {}".format("o" if human_player else "x"))
time_step = env.reset()
while not time_step.last():
player_id = time_step.observations["current_player"]
if player_id == human_player:
action = ask_user_for_action(time_step)
else:
obs = np.array(time_step.observations["info_state"][player_id])
if not args.old_api_stack:
action = np.argmax(
rl_module.forward_inference(
{"obs": torch.from_numpy(obs).unsqueeze(0).float()}
)["action_dist_inputs"][0].numpy()
)
else:
action = algo.compute_single_action(obs, policy_id="main")
# In case computer chooses an invalid action, pick a
# random one.
legal = time_step.observations["legal_actions"][player_id]
if action not in legal:
action = np.random.choice(legal)
time_step = env.step([action])
print(f"\n{env.get_state}")
print(f"\n{env.get_state}")
print("End of game!")
if time_step.rewards[human_player] > 0:
print("You win")
elif time_step.rewards[human_player] < 0:
print("You lose")
else:
print("Draw")
# Switch order of players
human_player = 1 - human_player
num_episodes += 1
algo.stop()
ray.shutdown()
rllib/examples/multi_agent/self_play_with_open_spiel.py
+247
View File
@@ -0,0 +1,247 @@
"""Example showing how one can implement a simple self-play training workflow.
Uses the open spiel adapter of RLlib with the "connect_four" game and
a multi-agent setup with a "main" policy and n "main_v[x]" policies
(x=version number), which are all at-some-point-frozen copies of
"main". At the very beginning, "main" plays against RandomPolicy.
Checks for the training progress after each training update via a custom
callback. We simply measure the win rate of "main" vs the opponent
("main_v[x]" or RandomPolicy at the beginning) by looking through the
achieved rewards in the episodes in the train batch. If this win rate
reaches some configurable threshold, we add a new policy to
the policy map (a frozen copy of the current "main" one) and change the
policy_mapping_fn to make new matches of "main" vs any of the previous
versions of "main" (including the just added one).
After training for n iterations, a configurable number of episodes can
be played by the user against the "main" agent on the command line.
"""
import functools
import numpy as np
import torch
from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
from ray.rllib.core.rl_module.multi_rl_module import MultiRLModuleSpec
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.env.utils import try_import_open_spiel, try_import_pyspiel
from ray.rllib.env.wrappers.open_spiel import OpenSpielEnv
from ray.rllib.examples._old_api_stack.policy.random_policy import RandomPolicy
from ray.rllib.examples.multi_agent.utils import (
SelfPlayCallback,
SelfPlayCallbackOldAPIStack,
ask_user_for_action,
)
from ray.rllib.examples.rl_modules.classes.random_rlm import RandomRLModule
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.rllib.policy.policy import PolicySpec
from ray.rllib.utils.metrics import NUM_ENV_STEPS_SAMPLED_LIFETIME
from ray.tune.registry import get_trainable_cls, register_env
from ray.tune.result import TRAINING_ITERATION
open_spiel = try_import_open_spiel(error=True)
pyspiel = try_import_pyspiel(error=True)
# Import after try_import_open_spiel, so we can error out with hints.
from open_spiel.python.rl_environment import Environment # noqa: E402
parser = add_rllib_example_script_args(default_timesteps=2000000)
parser.set_defaults(
env="connect_four",
checkpoint_freq=1,
checkpoint_at_end=True,
)
parser.add_argument(
"--win-rate-threshold",
type=float,
default=0.95,
help="Win-rate at which we setup another opponent by freezing the "
"current main policy and playing against a uniform distribution "
"of previously frozen 'main's from here on.",
)
parser.add_argument(
"--min-league-size",
type=float,
default=3,
help="Minimum number of policies/RLModules to consider the test passed. "
"The initial league size is 2: `main` and `random`. "
"`--min-league-size=3` thus means that one new policy/RLModule has been "
"added so far (b/c the `main` one has reached the `--win-rate-threshold "
"against the `random` Policy/RLModule).",
)
parser.add_argument(
"--num-episodes-human-play",
type=int,
default=10,
help="How many episodes to play against the user on the command "
"line after training has finished.",
)
parser.add_argument(
"--from-checkpoint",
type=str,
default=None,
help="Full path to a checkpoint file for restoring a previously saved "
"Algorithm state.",
)
if __name__ == "__main__":
args = parser.parse_args()
register_env("open_spiel_env", lambda _: OpenSpielEnv(pyspiel.load_game(args.env)))
def agent_to_module_mapping_fn(agent_id, episode, **kwargs):
# agent_id = [0|1] -> module depends on episode ID
# This way, we make sure that both modules sometimes play agent0
# (start player) and sometimes agent1 (player to move 2nd).
return "main" if hash(episode.id_) % 2 == agent_id else "random"
def policy_mapping_fn(agent_id, episode, worker, **kwargs):
# e.g. episode ID = 10234
# -> agent `0` -> main (b/c epsID % 2 == 0)
# -> agent `1` -> random (b/c epsID % 2 == 1)
return "main" if episode.episode_id % 2 == agent_id else "random"
config = (
get_trainable_cls(args.algo)
.get_default_config()
.environment("open_spiel_env")
# Set up the main piece in this experiment: The league-bases self-play
# callback, which controls adding new policies/Modules to the league and
# properly matching the different policies in the league with each other.
.callbacks(
functools.partial(
(
SelfPlayCallback
if not args.old_api_stack
else SelfPlayCallbackOldAPIStack
),
win_rate_threshold=args.win_rate_threshold,
)
)
.env_runners(
num_env_runners=(args.num_env_runners or 2),
num_envs_per_env_runner=1 if not args.old_api_stack else 5,
)
.multi_agent(
# Initial policy map: Random and default algo one. This will be expanded
# to more policy snapshots taken from "main" against which "main"
# will then play (instead of "random"). This is done in the
# custom callback defined above (`SelfPlayCallback`).
policies=(
{
# Our main policy, we'd like to optimize.
"main": PolicySpec(),
# An initial random opponent to play against.
"random": PolicySpec(policy_class=RandomPolicy),
}
if args.old_api_stack
else {"main", "random"}
),
# Assign agent 0 and 1 randomly to the "main" policy or
# to the opponent ("random" at first). Make sure (via episode_id)
# that "main" always plays against "random" (and not against
# another "main").
policy_mapping_fn=(
agent_to_module_mapping_fn
if not args.old_api_stack
else policy_mapping_fn
),
# Always just train the "main" policy.
policies_to_train=["main"],
)
.rl_module(
model_config=DefaultModelConfig(fcnet_hiddens=[512, 512]),
rl_module_spec=MultiRLModuleSpec(
rl_module_specs={
"main": RLModuleSpec(),
"random": RLModuleSpec(module_class=RandomRLModule),
}
),
)
)
# Only for PPO, change the `num_epochs` setting.
if args.algo == "PPO":
config.training(num_epochs=20)
stop = {
NUM_ENV_STEPS_SAMPLED_LIFETIME: args.stop_timesteps,
TRAINING_ITERATION: args.stop_iters,
"league_size": args.min_league_size,
}
# Train the "main" policy to play really well using self-play.
results = None
if not args.from_checkpoint:
results = run_rllib_example_script_experiment(
config, args, stop=stop, keep_ray_up=True
)
# Restore trained Algorithm (set to non-explore behavior) and play against
# human on command line.
if args.num_episodes_human_play > 0:
num_episodes = 0
config.explore = False
algo = config.build()
if args.from_checkpoint:
algo.restore(args.from_checkpoint)
else:
checkpoint = results.get_best_result().checkpoint
if not checkpoint:
raise ValueError("No last checkpoint found in results!")
algo.restore(checkpoint)
if not args.old_api_stack:
rl_module = algo.get_module("main")
# Play from the command line against the trained agent
# in an actual (non-RLlib-wrapped) open-spiel env.
human_player = 1
env = Environment(args.env)
while num_episodes < args.num_episodes_human_play:
print("You play as {}".format("o" if human_player else "x"))
time_step = env.reset()
while not time_step.last():
player_id = time_step.observations["current_player"]
if player_id == human_player:
action = ask_user_for_action(time_step)
else:
obs = np.array(time_step.observations["info_state"][player_id])
if not args.old_api_stack:
action = np.argmax(
rl_module.forward_inference(
{"obs": torch.from_numpy(obs).unsqueeze(0).float()}
)["action_dist_inputs"][0].numpy()
)
else:
action = algo.compute_single_action(obs, policy_id="main")
# In case computer chooses an invalid action, pick a
# random one.
legal = time_step.observations["legal_actions"][player_id]
if action not in legal:
action = np.random.choice(legal)
time_step = env.step([action])
print(f"\n{env.get_state}")
print(f"\n{env.get_state}")
print("End of game!")
if time_step.rewards[human_player] > 0:
print("You win")
elif time_step.rewards[human_player] < 0:
print("You lose")
else:
print("Draw")
# Switch order of players.
human_player = 1 - human_player
num_episodes += 1
algo.stop()
rllib/examples/multi_agent/shared_encoder_cartpole.py
+164
View File
@@ -0,0 +1,164 @@
"""A runnable example involving the use of a shared encoder module.
How to run this script
----------------------
`python [script file name].py --num-agents=2`
Control the number of agents and policies (RLModules) via --num-agents.
--encoder-emb-dim sets the encoder output dimension, and --no-shared-encoder
runs the experiment with independent encoders.
For debugging, use the following additional command line options
`--no-tune --num-env-runners=0`
which should allow you to set breakpoints anywhere in the RLlib code and
have the execution stop there for inspection and debugging.
For logging to your WandB account, use:
`--wandb-key=[your WandB API key] --wandb-project=[some project name]
--wandb-run-name=[optional: WandB run name (within the defined project)]`
Results to expect
-----------------
Under the shared encoder architecture, the target reward of 700 will typically be reached well before 100,000 iterations. A trial concludes as below:
+---------------------+------------+-----------------+--------+------------------+-------+-------------------+-------------+-------------+
| Trial name | status | loc | iter | total time (s) | ts | combined return | return p1 | return p0 |
|---------------------+------------+-----------------+--------+------------------+-------+-------------------+-------------+-------------|
| VPG_env_ab318_00000 | TERMINATED | 127.0.0.1:37375 | 33 | 44.2689 | 74197 | 611.35 | 191.71 | 419.64 |
+---------------------+------------+-----------------+--------+------------------+-------+-------------------+-------------+-------------+
Without a shared encoder, a lower reward is typically achieved after training for the full 100,000 timesteps:
+---------------------+------------+-----------------+--------+------------------+--------+-------------------+-------------+-------------+
| Trial name | status | loc | iter | total time (s) | ts | combined return | return p0 | return p1 |
|---------------------+------------+-----------------+--------+------------------+--------+-------------------+-------------+-------------|
| VPG_env_2e79e_00000 | TERMINATED | 127.0.0.1:39076 | 37 | 52.127 | 103894 | 526.66 | 85.78 | 440.88 |
+---------------------+------------+-----------------+--------+------------------+--------+-------------------+-------------+-------------+
"""
import gymnasium as gym
from ray.rllib.core.rl_module.multi_rl_module import MultiRLModuleSpec
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.examples.algorithms.classes.vpg import VPGConfig
from ray.rllib.examples.envs.classes.multi_agent import MultiAgentCartPole
from ray.rllib.examples.learners.classes.vpg_torch_learner_shared_optimizer import (
VPGTorchLearnerSharedOptimizer,
)
from ray.rllib.examples.rl_modules.classes.vpg_using_shared_encoder_rlm import (
SHARED_ENCODER_ID,
SharedEncoder,
VPGMultiRLModuleWithSharedEncoder,
VPGPolicyAfterSharedEncoder,
VPGPolicyNoSharedEncoder,
)
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.tune.registry import register_env
parser = add_rllib_example_script_args(
default_iters=200,
default_timesteps=100000,
default_reward=600.0,
)
parser.set_defaults(
algo="VPG",
num_agents=2,
)
parser.add_argument("--encoder-emb-dim", type=int, default=64)
parser.add_argument("--no-shared-encoder", action="store_true")
if __name__ == "__main__":
args = parser.parse_args()
assert args.algo == "VPG", "The shared encoder example is meant for VPG agents."
assert args.num_agents == 2, "This example makes use of two agents."
single_agent_env = gym.make(
"CartPole-v1"
) # To allow instantiation of shared encoder
EMBEDDING_DIM = args.encoder_emb_dim # encoder output dim
if args.no_shared_encoder:
print("Running experiment without shared encoder")
specs = MultiRLModuleSpec(
rl_module_specs={
# Large policy net.
"p0": RLModuleSpec(
module_class=VPGPolicyNoSharedEncoder,
model_config={
"embedding_dim": EMBEDDING_DIM,
"hidden_dim": 64,
},
),
# Small policy net.
"p1": RLModuleSpec(
module_class=VPGPolicyNoSharedEncoder,
model_config={
"embedding_dim": EMBEDDING_DIM,
"hidden_dim": 64,
},
),
}
)
else:
specs = MultiRLModuleSpec(
multi_rl_module_class=VPGMultiRLModuleWithSharedEncoder,
rl_module_specs={
# Shared encoder.
SHARED_ENCODER_ID: RLModuleSpec(
module_class=SharedEncoder,
model_config={"embedding_dim": EMBEDDING_DIM},
observation_space=single_agent_env.observation_space,
action_space=single_agent_env.action_space,
),
# Large policy net.
"p0": RLModuleSpec(
module_class=VPGPolicyAfterSharedEncoder,
model_config={
"embedding_dim": EMBEDDING_DIM,
"hidden_dim": 64,
},
),
# Small policy net.
"p1": RLModuleSpec(
module_class=VPGPolicyAfterSharedEncoder,
model_config={
"embedding_dim": EMBEDDING_DIM,
"hidden_dim": 64,
},
),
},
)
# Register our environment with tune.
register_env(
"env",
lambda _: MultiAgentCartPole(config={"num_agents": args.num_agents}),
)
base_config = (
VPGConfig()
.environment("env" if args.num_agents > 0 else "CartPole-v1")
.training(
learner_class=VPGTorchLearnerSharedOptimizer
if not args.no_shared_encoder
else None,
train_batch_size=2048,
lr=1e-2,
)
.multi_agent(
policies={"p0", "p1"},
policy_mapping_fn=lambda agent_id, episode, **kw: f"p{agent_id}",
)
.rl_module(
rl_module_spec=specs,
)
)
run_rllib_example_script_experiment(base_config, args)
rllib/examples/multi_agent/two_step_game_with_grouped_agents.py
+90
View File
@@ -0,0 +1,90 @@
"""The two-step game from the QMIX paper:
https://arxiv.org/pdf/1803.11485.pdf
See also: rllib/examples/centralized_critic.py for centralized critic PPO on this game.
How to run this script
----------------------
`python [script file name].py --num-agents=2`
Note that in this script, we use an multi-agent environment in which both
agents that normally play this game have been merged into one agent with ID
"agents" and observation- and action-spaces being 2-tupled (1 item for each
agent). The "agents" agent is mapped to the policy with ID "p0".
For debugging, use the following additional command line options
`--no-tune --num-env-runners=0`
Which should allow you to set breakpoints anywhere in the RLlib code and
have the execution stop there for inspection and debugging.
For logging to your WandB account, use:
`--wandb-key=[your WandB API key] --wandb-project=[some project name]
--wandb-run-name=[optional: WandB run name (within the defined project)]`
Results to expect
-----------------
You should expect a reward of 8.0 (the max to reach in thie game) eventually
being achieved by a simple PPO policy (no tuning, just using RLlib's default settings):
+---------------------------------+------------+-----------------+--------+
| Trial name | status | loc | iter |
|---------------------------------+------------+-----------------+--------+
| PPO_grouped_twostep_4354b_00000 | TERMINATED | 127.0.0.1:42602 | 20 |
+---------------------------------+------------+-----------------+--------+
+------------------+-------+-------------------+-------------+
| total time (s) | ts | combined reward | reward p0 |
+------------------+-------+-------------------+-------------|
| 87.5756 | 80000 | 8 | 8 |
+------------------+-------+-------------------+-------------+
"""
from ray.rllib.connectors.env_to_module import FlattenObservations
from ray.rllib.core.rl_module.multi_rl_module import MultiRLModuleSpec
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.examples.envs.classes.multi_agent.two_step_game import (
TwoStepGameWithGroupedAgents,
)
from ray.rllib.examples.utils import (
add_rllib_example_script_args,
run_rllib_example_script_experiment,
)
from ray.tune.registry import get_trainable_cls, register_env
parser = add_rllib_example_script_args(default_reward=7.0)
if __name__ == "__main__":
args = parser.parse_args()
assert args.num_agents == 2, "Must set --num-agents=2 when running this script!"
register_env(
"grouped_twostep",
lambda config: TwoStepGameWithGroupedAgents(config),
)
base_config = (
get_trainable_cls(args.algo)
.get_default_config()
.environment("grouped_twostep")
.env_runners(
env_to_module_connector=lambda env, spaces, device: FlattenObservations(
multi_agent=True
),
)
.multi_agent(
policies={"p0"},
policy_mapping_fn=lambda aid, *a, **kw: "p0",
)
.rl_module(
rl_module_spec=MultiRLModuleSpec(
rl_module_specs={
"p0": RLModuleSpec(),
},
)
)
)
run_rllib_example_script_experiment(base_config, args)
rllib/examples/multi_agent/utils/__init__.py
+43
View File
@@ -0,0 +1,43 @@
import sys
from ray.rllib.examples.multi_agent.utils.self_play_callback import SelfPlayCallback
from ray.rllib.examples.multi_agent.utils.self_play_callback_old_api_stack import (
SelfPlayCallbackOldAPIStack,
)
from ray.rllib.examples.multi_agent.utils.self_play_league_based_callback import (
SelfPlayLeagueBasedCallback,
)
from ray.rllib.examples.multi_agent.utils.self_play_league_based_callback_old_api_stack import ( # noqa
SelfPlayLeagueBasedCallbackOldAPIStack,
)
def ask_user_for_action(time_step):
"""Asks the user for a valid action on the command line and returns it.
Re-queries the user until she picks a valid one.
Args:
time_step: The open spiel Environment time-step object.
"""
pid = time_step.observations["current_player"]
legal_moves = time_step.observations["legal_actions"][pid]
choice = -1
while choice not in legal_moves:
print("Choose an action from {}:".format(legal_moves))
sys.stdout.flush()
choice_str = input()
try:
choice = int(choice_str)
except ValueError:
continue
return choice
__all__ = [
"ask_user_for_action",
"SelfPlayCallback",
"SelfPlayLeagueBasedCallback",
"SelfPlayCallbackOldAPIStack",
"SelfPlayLeagueBasedCallbackOldAPIStack",
]
rllib/examples/multi_agent/utils/self_play_callback.py
+97
View File
@@ -0,0 +1,97 @@
from collections import defaultdict
import numpy as np
from ray.rllib.callbacks.callbacks import RLlibCallback
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS
class SelfPlayCallback(RLlibCallback):
def __init__(self, win_rate_threshold):
super().__init__()
# 0=RandomPolicy, 1=1st main policy snapshot,
# 2=2nd main policy snapshot, etc..
self.current_opponent = 0
self.win_rate_threshold = win_rate_threshold
# Report the matchup counters (who played against whom?).
self._matching_stats = defaultdict(int)
def on_episode_end(
self,
*,
episode,
env_runner,
metrics_logger,
env,
env_index,
rl_module,
**kwargs,
) -> None:
# Compute the win rate for this episode and log it with a window of 100.
main_agent = 0 if episode.module_for(0) == "main" else 1
rewards = episode.get_rewards()
if main_agent in rewards:
main_won = rewards[main_agent][-1] == 1.0
metrics_logger.log_value(
"win_rate",
main_won,
reduce="mean",
window=100,
)
def on_train_result(self, *, algorithm, metrics_logger=None, result, **kwargs):
win_rate = result[ENV_RUNNER_RESULTS]["win_rate"]
print(f"Iter={algorithm.iteration} win-rate={win_rate} -> ", end="")
# If win rate is good -> Snapshot current policy and play against
# it next, keeping the snapshot fixed and only improving the "main"
# policy.
if win_rate > self.win_rate_threshold:
self.current_opponent += 1
new_module_id = f"main_v{self.current_opponent}"
print(f"adding new opponent to the mix ({new_module_id}).")
# Re-define the mapping function, such that "main" is forced
# to play against any of the previously played modules
# (excluding "random").
def agent_to_module_mapping_fn(agent_id, episode, **kwargs):
# agent_id = [0|1] -> policy depends on episode ID
# This way, we make sure that both modules sometimes play
# (start player) and sometimes agent1 (player to move 2nd).
opponent = "main_v{}".format(
np.random.choice(list(range(1, self.current_opponent + 1)))
)
if hash(episode.id_) % 2 == agent_id:
self._matching_stats[("main", opponent)] += 1
return "main"
else:
return opponent
main_module = algorithm.get_module("main")
algorithm.add_module(
module_id=new_module_id,
module_spec=RLModuleSpec.from_module(main_module),
new_agent_to_module_mapping_fn=agent_to_module_mapping_fn,
)
# TODO (sven): Maybe we should move this convenience step back into
# `Algorithm.add_module()`? Would be less explicit, but also easier.
algorithm.set_state(
{
"learner_group": {
"learner": {
"rl_module": {
new_module_id: main_module.get_state(),
}
}
}
}
)
else:
print("not good enough; will keep learning ...")
# +2 = main + random
result["league_size"] = self.current_opponent + 2
print(f"Matchups:\n{self._matching_stats}")
rllib/examples/multi_agent/utils/self_play_callback_old_api_stack.py
+78
View File
@@ -0,0 +1,78 @@
import numpy as np
from ray._common.deprecation import Deprecated
from ray.rllib.callbacks.callbacks import RLlibCallback
from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS
@Deprecated(help="Use the example for the new RLlib API stack.", error=False)
class SelfPlayCallbackOldAPIStack(RLlibCallback):
def __init__(self, win_rate_threshold):
super().__init__()
# 0=RandomPolicy, 1=1st main policy snapshot,
# 2=2nd main policy snapshot, etc..
self.current_opponent = 0
self.win_rate_threshold = win_rate_threshold
def on_train_result(self, *, algorithm, result, **kwargs):
# Get the win rate for the train batch.
# Note that normally, you should set up a proper evaluation config,
# such that evaluation always happens on the already updated policy,
# instead of on the already used train_batch.
main_rew = result[ENV_RUNNER_RESULTS]["hist_stats"].pop("policy_main_reward")
opponent_rew = list(result[ENV_RUNNER_RESULTS]["hist_stats"].values())[0]
assert len(main_rew) == len(opponent_rew)
won = 0
for r_main, r_opponent in zip(main_rew, opponent_rew):
if r_main > r_opponent:
won += 1
win_rate = won / len(main_rew)
result["win_rate"] = win_rate
print(f"Iter={algorithm.iteration} win-rate={win_rate} -> ", end="")
# If win rate is good -> Snapshot current policy and play against
# it next, keeping the snapshot fixed and only improving the "main"
# policy.
if win_rate > self.win_rate_threshold:
self.current_opponent += 1
new_pol_id = f"main_v{self.current_opponent}"
print(f"adding new opponent to the mix ({new_pol_id}).")
# Re-define the mapping function, such that "main" is forced
# to play against any of the previously played policies
# (excluding "random").
def policy_mapping_fn(agent_id, episode, worker, **kwargs):
# agent_id = [0|1] -> policy depends on episode ID
# This way, we make sure that both policies sometimes play
# (start player) and sometimes agent1 (player to move 2nd).
return (
"main"
if episode.episode_id % 2 == agent_id
else "main_v{}".format(
np.random.choice(list(range(1, self.current_opponent + 1)))
)
)
main_policy = algorithm.get_policy("main")
new_policy = algorithm.add_policy(
policy_id=new_pol_id,
policy_cls=type(main_policy),
policy_mapping_fn=policy_mapping_fn,
config=main_policy.config,
observation_space=main_policy.observation_space,
action_space=main_policy.action_space,
)
# Set the weights of the new policy to the main policy.
# We'll keep training the main policy, whereas `new_pol_id` will
# remain fixed.
main_state = main_policy.get_state()
new_policy.set_state(main_state)
# We need to sync the just copied local weights (from main policy)
# to all the remote workers as well.
algorithm.env_runner_group.sync_weights()
else:
print("not good enough; will keep learning ...")
# +2 = main + random
result["league_size"] = self.current_opponent + 2
rllib/examples/multi_agent/utils/self_play_league_based_callback.py
+299
View File
@@ -0,0 +1,299 @@
import re
from collections import defaultdict
from pprint import pprint
import numpy as np
from ray.rllib.callbacks.callbacks import RLlibCallback
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS
class SelfPlayLeagueBasedCallback(RLlibCallback):
def __init__(self, win_rate_threshold):
super().__init__()
# All policies in the league.
self.main_policies = {"main", "main_0"}
self.main_exploiters = {"main_exploiter_0", "main_exploiter_1"}
self.league_exploiters = {"league_exploiter_0", "league_exploiter_1"}
# Set of currently trainable policies in the league.
self.trainable_policies = {"main"}
# Set of currently non-trainable (frozen) policies in the league.
self.non_trainable_policies = {
"main_0",
"league_exploiter_0",
"main_exploiter_0",
}
# The win-rate value reaching of which leads to a new module being added
# to the leage (frozen copy of main).
self.win_rate_threshold = win_rate_threshold
# Store the win rates for league overview printouts.
self.win_rates = {}
# Report the matchup counters (who played against whom?).
self._matching_stats = defaultdict(int)
def on_episode_end(
self,
*,
episode,
env_runner,
metrics_logger,
env,
env_index,
rl_module,
**kwargs,
) -> None:
num_learning_policies = (
episode.module_for(0) in env_runner.config.policies_to_train
) + (episode.module_for(1) in env_runner.config.policies_to_train)
# Make sure the mapping function doesn't match two non-trainables together.
# This would be a waste of EnvRunner resources.
# assert num_learning_policies > 0
# Ignore matches between two learning policies and don't count win-rates for
# these.
assert num_learning_policies > 0, (
f"agent=0 -> mod={episode.module_for(0)}; "
f"agent=1 -> mod={episode.module_for(1)}; "
f"EnvRunner.config.policies_to_train={env_runner.config.policies_to_train}"
)
if num_learning_policies == 1:
# Compute the win rate for this episode (only looking at non-trained
# opponents, such as random or frozen policies) and log it with some window.
rewards_dict = episode.get_rewards()
for aid, rewards in rewards_dict.items():
mid = episode.module_for(aid)
won = rewards[-1] == 1.0
metrics_logger.log_value(
f"win_rate_{mid}",
won,
window=100,
)
def on_train_result(self, *, algorithm, metrics_logger=None, result, **kwargs):
local_worker = algorithm.env_runner
# Avoid `self` being pickled into the remote function below.
_trainable_policies = self.trainable_policies
# Get the win rate for the train batch.
# Note that normally, one should set up a proper evaluation config,
# such that evaluation always happens on the already updated policy,
# instead of on the already used train_batch.
league_changed = False
keys = [
k for k in result[ENV_RUNNER_RESULTS].keys() if k.startswith("win_rate_")
]
for key in keys:
module_id = key[9:]
self.win_rates[module_id] = result[ENV_RUNNER_RESULTS][key]
# Policy is frozen; ignore.
if module_id in self.non_trainable_policies:
continue
print(
f"Iter={algorithm.iteration} {module_id}'s "
f"win-rate={self.win_rates[module_id]} -> ",
end="",
)
# If win rate is good -> Snapshot current policy and decide,
# whether to freeze the copy or not.
if self.win_rates[module_id] > self.win_rate_threshold:
is_main = re.match("^main(_\\d+)?$", module_id)
initializing_exploiters = False
# First time, main manages a decent win-rate against random:
# Add league_exploiter_1 and main_exploiter_1 as trainables to the mix.
if is_main and len(self.trainable_policies) == 1:
initializing_exploiters = True
self.trainable_policies.add("league_exploiter_1")
self.trainable_policies.add("main_exploiter_1")
# If main manages to win (above threshold) against the entire league
# -> increase the league by another frozen copy of main,
# main-exploiters or league-exploiters.
else:
keep_training = (
False
if is_main
else np.random.choice([True, False], p=[0.3, 0.7])
)
if module_id in self.main_policies:
new_mod_id = re.sub(
"(main)(_\\d+)?$",
f"\\1_{len(self.main_policies) - 1}",
module_id,
)
self.main_policies.add(new_mod_id)
elif module_id in self.main_exploiters:
new_mod_id = re.sub(
"_\\d+$", f"_{len(self.main_exploiters)}", module_id
)
self.main_exploiters.add(new_mod_id)
else:
new_mod_id = re.sub(
"_\\d+$", f"_{len(self.league_exploiters)}", module_id
)
self.league_exploiters.add(new_mod_id)
if keep_training:
self.trainable_policies.add(new_mod_id)
else:
self.non_trainable_policies.add(new_mod_id)
print(f"adding new opponents to the mix ({new_mod_id}).")
# Initialize state variablers for agent-to-module mapping. Note, we
# need to keep track of the league-exploiter to always match a
# non-trainable policy with a trainable one - otherwise matches are
# a waste of resources.
self.type_count = 0
self.exploiter = None
def agent_to_module_mapping_fn(agent_id, episode, **kwargs):
# Pick whether this is ...
type_ = np.random.choice([1, 2])
# Each second third call reset state variables. Note, there will
# be always two agents playing against each others.
if self.type_count >= 2:
# Reset the counter.
self.type_count = 0
# Set the exploiter to `None`.
self.exploiter = None
# Increment the counter for each agent.
self.type_count += 1
# 1) League exploiter vs any other.
if type_ == 1:
# Note, the exploiter could be either of `type_==1` or `type_==2`.
if not self.exploiter:
self.exploiter = "league_exploiter_" + str(
np.random.choice(
list(range(len(self.league_exploiters)))
)
)
# This league exploiter is frozen: Play against a
# trainable policy.
if self.exploiter not in self.trainable_policies:
opponent = np.random.choice(list(self.trainable_policies))
# League exploiter is trainable: Play against any other
# non-trainable policy.
else:
opponent = np.random.choice(
list(self.non_trainable_policies)
)
# Only record match stats once per match.
if hash(episode.id_) % 2 == agent_id:
self._matching_stats[(self.exploiter, opponent)] += 1
return self.exploiter
else:
return opponent
# 2) Main exploiter vs main.
else:
# Note, the exploiter could be either of `type_==1` or `type_==2`.
if not self.exploiter:
self.exploiter = "main_exploiter_" + str(
np.random.choice(list(range(len(self.main_exploiters))))
)
# Main exploiter is frozen: Play against the main
# policy.
if self.exploiter not in self.trainable_policies:
main = "main"
# Main exploiter is trainable: Play against any
# frozen main.
else:
main = np.random.choice(list(self.main_policies - {"main"}))
# Only record match stats once per match.
if hash(episode.id_) % 2 == agent_id:
self._matching_stats[(self.exploiter, main)] += 1
return self.exploiter
else:
return main
multi_rl_module = local_worker.module
main_module = multi_rl_module["main"]
# Set the weights of the new polic(y/ies).
if initializing_exploiters:
main_state = main_module.get_state()
multi_rl_module["main_0"].set_state(main_state)
multi_rl_module["league_exploiter_1"].set_state(main_state)
multi_rl_module["main_exploiter_1"].set_state(main_state)
# We need to sync the just copied local weights to all the
# remote workers and remote Learner workers as well.
algorithm.env_runner_group.sync_weights(
policies=["main_0", "league_exploiter_1", "main_exploiter_1"]
)
algorithm.learner_group.set_weights(multi_rl_module.get_state())
else:
algorithm.add_module(
module_id=new_mod_id,
module_spec=RLModuleSpec.from_module(main_module),
)
# TODO (sven): Maybe we should move this convenience step back into
# `Algorithm.add_module()`? Would be less explicit, but also
# easier.
algorithm.set_state(
{
"learner_group": {
"learner": {
"rl_module": {
new_mod_id: multi_rl_module[
module_id
].get_state(),
}
}
}
}
)
algorithm.env_runner_group.foreach_env_runner(
lambda env_runner: env_runner.config.multi_agent(
policy_mapping_fn=agent_to_module_mapping_fn,
# This setting doesn't really matter for EnvRunners (no
# training going on there, but we'll update this as well
# here for good measure).
policies_to_train=_trainable_policies,
),
local_env_runner=True,
)
# Set all Learner workers' should_module_be_updated to the new
# value.
algorithm.learner_group.foreach_learner(
func=lambda learner: learner.config.multi_agent(
policies_to_train=_trainable_policies,
),
timeout_seconds=0.0, # fire-and-forget
)
league_changed = True
else:
print("not good enough; will keep learning ...")
# Add current league size to results dict.
result["league_size"] = len(self.non_trainable_policies) + len(
self.trainable_policies
)
if league_changed:
self._print_league()
def _print_league(self):
print("--- League ---")
print("Matchups:")
pprint(self._matching_stats)
print("Trainable policies (win-rates):")
for p in sorted(self.trainable_policies):
wr = self.win_rates[p] if p in self.win_rates else 0.0
print(f"\t{p}: {wr}")
print("Frozen policies:")
for p in sorted(self.non_trainable_policies):
wr = self.win_rates[p] if p in self.win_rates else 0.0
print(f"\t{p}: {wr}")
print()
rllib/examples/multi_agent/utils/self_play_league_based_callback_old_api_stack.py
+201
View File
@@ -0,0 +1,201 @@
import re
import numpy as np
from ray._common.deprecation import Deprecated
from ray.rllib.callbacks.callbacks import RLlibCallback
from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS
@Deprecated(help="Use the example for the new RLlib API stack", error=False)
class SelfPlayLeagueBasedCallbackOldAPIStack(RLlibCallback):
def __init__(self, win_rate_threshold):
super().__init__()
# All policies in the league.
self.main_policies = {"main", "main_0"}
self.main_exploiters = {"main_exploiter_0", "main_exploiter_1"}
self.league_exploiters = {"league_exploiter_0", "league_exploiter_1"}
# Set of currently trainable policies in the league.
self.trainable_policies = {"main"}
# Set of currently non-trainable (frozen) policies in the league.
self.non_trainable_policies = {
"main_0",
"league_exploiter_0",
"main_exploiter_0",
}
# The win-rate value reaching of which leads to a new module being added
# to the leage (frozen copy of main).
self.win_rate_threshold = win_rate_threshold
# Store the win rates for league overview printouts.
self.win_rates = {}
def on_train_result(self, *, algorithm, result, **kwargs):
# Avoid `self` being pickled into the remote function below.
_trainable_policies = self.trainable_policies
# Get the win rate for the train batch.
# Note that normally, you should set up a proper evaluation config,
# such that evaluation always happens on the already updated policy,
# instead of on the already used train_batch.
for policy_id, rew in result[ENV_RUNNER_RESULTS]["hist_stats"].items():
mo = re.match("^policy_(.+)_reward$", policy_id)
if mo is None:
continue
policy_id = mo.group(1)
# Calculate this policy's win rate.
won = 0
for r in rew:
if r > 0.0: # win = 1.0; loss = -1.0
won += 1
win_rate = won / len(rew)
self.win_rates[policy_id] = win_rate
# Policy is frozen; ignore.
if policy_id in self.non_trainable_policies:
continue
print(
f"Iter={algorithm.iteration} {policy_id}'s " f"win-rate={win_rate} -> ",
end="",
)
# If win rate is good -> Snapshot current policy and decide,
# whether to freeze the copy or not.
if win_rate > self.win_rate_threshold:
is_main = re.match("^main(_\\d+)?$", policy_id)
initializing_exploiters = False
# First time, main manages a decent win-rate against random:
# Add league_exploiter_0 and main_exploiter_0 to the mix.
if is_main and len(self.trainable_policies) == 1:
initializing_exploiters = True
self.trainable_policies.add("league_exploiter_0")
self.trainable_policies.add("main_exploiter_0")
else:
keep_training = (
False
if is_main
else np.random.choice([True, False], p=[0.3, 0.7])
)
if policy_id in self.main_policies:
new_pol_id = re.sub(
"_\\d+$", f"_{len(self.main_policies) - 1}", policy_id
)
self.main_policies.add(new_pol_id)
elif policy_id in self.main_exploiters:
new_pol_id = re.sub(
"_\\d+$", f"_{len(self.main_exploiters)}", policy_id
)
self.main_exploiters.add(new_pol_id)
else:
new_pol_id = re.sub(
"_\\d+$", f"_{len(self.league_exploiters)}", policy_id
)
self.league_exploiters.add(new_pol_id)
if keep_training:
self.trainable_policies.add(new_pol_id)
else:
self.non_trainable_policies.add(new_pol_id)
print(f"adding new opponents to the mix ({new_pol_id}).")
# Update our mapping function accordingly.
def policy_mapping_fn(agent_id, episode, worker=None, **kwargs):
# Pick, whether this is ...
type_ = np.random.choice([1, 2])
# 1) League exploiter vs any other.
if type_ == 1:
league_exploiter = "league_exploiter_" + str(
np.random.choice(list(range(len(self.league_exploiters))))
)
# This league exploiter is frozen: Play against a
# trainable policy.
if league_exploiter not in self.trainable_policies:
opponent = np.random.choice(list(self.trainable_policies))
# League exploiter is trainable: Play against any other
# non-trainable policy.
else:
opponent = np.random.choice(
list(self.non_trainable_policies)
)
print(f"{league_exploiter} vs {opponent}")
return (
league_exploiter
if episode.episode_id % 2 == agent_id
else opponent
)
# 2) Main exploiter vs main.
else:
main_exploiter = "main_exploiter_" + str(
np.random.choice(list(range(len(self.main_exploiters))))
)
# Main exploiter is frozen: Play against the main
# policy.
if main_exploiter not in self.trainable_policies:
main = "main"
# Main exploiter is trainable: Play against any
# frozen main.
else:
main = np.random.choice(list(self.main_policies - {"main"}))
# print(f"{main_exploiter} vs {main}")
return (
main_exploiter
if episode.episode_id % 2 == agent_id
else main
)
# Set the weights of the new polic(y/ies).
if initializing_exploiters:
main_state = algorithm.get_policy("main").get_state()
pol_map = algorithm.env_runner.policy_map
pol_map["main_0"].set_state(main_state)
pol_map["league_exploiter_1"].set_state(main_state)
pol_map["main_exploiter_1"].set_state(main_state)
# We need to sync the just copied local weights to all the
# remote workers as well.
algorithm.env_runner_group.sync_weights(
policies=["main_0", "league_exploiter_1", "main_exploiter_1"]
)
def _set(worker):
worker.set_policy_mapping_fn(policy_mapping_fn)
worker.set_is_policy_to_train(_trainable_policies)
algorithm.env_runner_group.foreach_env_runner(_set)
else:
base_pol = algorithm.get_policy(policy_id)
new_policy = algorithm.add_policy(
policy_id=new_pol_id,
policy_cls=type(base_pol),
policy_mapping_fn=policy_mapping_fn,
policies_to_train=self.trainable_policies,
config=base_pol.config,
observation_space=base_pol.observation_space,
action_space=base_pol.action_space,
)
main_state = base_pol.get_state()
new_policy.set_state(main_state)
# We need to sync the just copied local weights to all the
# remote workers as well.
algorithm.env_runner_group.sync_weights(policies=[new_pol_id])
self._print_league()
else:
print("not good enough; will keep learning ...")
def _print_league(self):
print("--- League ---")
print("Trainable policies (win-rates):")
for p in sorted(self.trainable_policies):
wr = self.win_rates[p] if p in self.win_rates else 0.0
print(f"\t{p}: {wr}")
print("Frozen policies:")
for p in sorted(self.non_trainable_policies):
wr = self.win_rates[p] if p in self.win_rates else 0.0
print(f"\t{p}: {wr}")
print()