chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,274 @@
|
||||
import unittest
|
||||
|
||||
import ray
|
||||
import ray.rllib.algorithms.appo as appo
|
||||
from ray.rllib.algorithms.impala.impala import LEARNER_RESULTS_CURR_ENTROPY_COEFF_KEY
|
||||
from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
|
||||
from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
|
||||
from ray.rllib.utils.metrics import (
|
||||
ENV_RUNNER_RESULTS,
|
||||
LEARNER_RESULTS,
|
||||
NUM_ENV_STEPS_SAMPLED_LIFETIME,
|
||||
)
|
||||
from ray.rllib.utils.test_utils import (
|
||||
check_compute_single_action,
|
||||
check_train_results,
|
||||
check_train_results_new_api_stack,
|
||||
)
|
||||
|
||||
|
||||
class TestAPPO(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
ray.init()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
ray.shutdown()
|
||||
|
||||
def test_appo_compilation(self):
|
||||
"""Test whether APPO can be built with both frameworks."""
|
||||
config = (
|
||||
appo.APPOConfig().environment("CartPole-v1").env_runners(num_env_runners=1)
|
||||
)
|
||||
algo = config.build()
|
||||
|
||||
num_iterations = 2
|
||||
for i in range(num_iterations):
|
||||
results = algo.train()
|
||||
print(results)
|
||||
check_train_results_new_api_stack(results)
|
||||
|
||||
algo.stop()
|
||||
|
||||
def test_appo_compilation_use_kl_loss(self):
|
||||
"""Test whether APPO can be built with kl_loss enabled."""
|
||||
config = (
|
||||
appo.APPOConfig().env_runners(num_env_runners=1).training(use_kl_loss=True)
|
||||
)
|
||||
num_iterations = 2
|
||||
|
||||
algo = config.build(env="CartPole-v1")
|
||||
for i in range(num_iterations):
|
||||
results = algo.train()
|
||||
print(results)
|
||||
check_train_results_new_api_stack(results)
|
||||
algo.stop()
|
||||
|
||||
def test_appo_two_optimizers_two_lrs(self):
|
||||
# Not explicitly setting this should cause a warning, but not fail.
|
||||
# config["_tf_policy_handles_more_than_one_loss"] = True
|
||||
config = (
|
||||
appo.APPOConfig()
|
||||
.api_stack(
|
||||
enable_rl_module_and_learner=False,
|
||||
enable_env_runner_and_connector_v2=False,
|
||||
)
|
||||
.env_runners(num_env_runners=1)
|
||||
.training(
|
||||
_separate_vf_optimizer=True,
|
||||
_lr_vf=0.002,
|
||||
# Make sure we have two completely separate models for policy and
|
||||
# value function.
|
||||
model={
|
||||
"vf_share_layers": False,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
num_iterations = 2
|
||||
|
||||
# Only supported for tf so far.
|
||||
algo = config.build(env="CartPole-v1")
|
||||
for i in range(num_iterations):
|
||||
results = algo.train()
|
||||
check_train_results(results)
|
||||
print(results)
|
||||
check_compute_single_action(algo)
|
||||
algo.stop()
|
||||
|
||||
def test_appo_entropy_coeff_schedule(self):
|
||||
config = (
|
||||
appo.APPOConfig()
|
||||
.environment("CartPole-v1")
|
||||
.env_runners(
|
||||
num_env_runners=1,
|
||||
rollout_fragment_length=10,
|
||||
)
|
||||
.training(
|
||||
train_batch_size_per_learner=20,
|
||||
entropy_coeff=[
|
||||
[0, 0.1],
|
||||
[50000, 0.01],
|
||||
],
|
||||
)
|
||||
.reporting(
|
||||
min_train_timesteps_per_iteration=20,
|
||||
# 0 metrics reporting delay, this makes sure timestep,
|
||||
# which entropy coeff depends on, is updated after each worker rollout.
|
||||
min_time_s_per_iteration=0,
|
||||
)
|
||||
)
|
||||
|
||||
def _step_n_times(algo, n: int):
|
||||
"""Step Algorithm n times.
|
||||
|
||||
Returns:
|
||||
learning rate at the end of the execution.
|
||||
"""
|
||||
for _ in range(n):
|
||||
results = algo.train()
|
||||
print(results)
|
||||
return results[LEARNER_RESULTS][DEFAULT_POLICY_ID][
|
||||
LEARNER_RESULTS_CURR_ENTROPY_COEFF_KEY
|
||||
]
|
||||
|
||||
algo = config.build()
|
||||
|
||||
coeff = _step_n_times(algo, 10)
|
||||
# Should be close to the starting coeff of 0.01.
|
||||
ts_sampled = algo.metrics.peek(
|
||||
(ENV_RUNNER_RESULTS, NUM_ENV_STEPS_SAMPLED_LIFETIME)
|
||||
)
|
||||
expected_coeff = 0.1 - ((0.1 - 0.01) / 50000 * ts_sampled)
|
||||
self.assertLessEqual(coeff, expected_coeff + 0.005)
|
||||
self.assertGreaterEqual(coeff, expected_coeff - 0.005)
|
||||
|
||||
coeff = _step_n_times(algo, 20)
|
||||
ts_sampled = algo.metrics.peek(
|
||||
(ENV_RUNNER_RESULTS, NUM_ENV_STEPS_SAMPLED_LIFETIME)
|
||||
)
|
||||
expected_coeff = 0.1 - ((0.1 - 0.01) / 50000 * ts_sampled)
|
||||
self.assertLessEqual(coeff, expected_coeff + 0.005)
|
||||
self.assertGreaterEqual(coeff, expected_coeff - 0.005)
|
||||
|
||||
algo.stop()
|
||||
|
||||
def test_appo_learning_rate_schedule(self):
|
||||
config = (
|
||||
appo.APPOConfig()
|
||||
.env_runners(
|
||||
num_env_runners=1,
|
||||
batch_mode="truncate_episodes",
|
||||
rollout_fragment_length=10,
|
||||
)
|
||||
.training(
|
||||
train_batch_size_per_learner=20,
|
||||
entropy_coeff=0.01,
|
||||
# Setup lr schedule for testing.
|
||||
lr=[[0, 5e-2], [50000, 0.0]],
|
||||
)
|
||||
.reporting(
|
||||
min_train_timesteps_per_iteration=20,
|
||||
# 0 metrics reporting delay, this makes sure timestep,
|
||||
# which entropy coeff depends on, is updated after each worker rollout.
|
||||
min_time_s_per_iteration=0,
|
||||
)
|
||||
)
|
||||
|
||||
def _step_n_times(algo, n: int):
|
||||
"""Step Algorithm n times.
|
||||
|
||||
Returns:
|
||||
learning rate at the end of the execution.
|
||||
"""
|
||||
for _ in range(n):
|
||||
results = algo.train()
|
||||
print(results)
|
||||
return results[LEARNER_RESULTS][DEFAULT_POLICY_ID][
|
||||
"default_optimizer_learning_rate"
|
||||
]
|
||||
|
||||
algo = config.build(env="CartPole-v1")
|
||||
|
||||
lr1 = _step_n_times(algo, 10)
|
||||
lr2 = _step_n_times(algo, 10)
|
||||
|
||||
self.assertGreater(lr1, lr2)
|
||||
|
||||
algo.stop()
|
||||
|
||||
def test_appo_model_variables(self):
|
||||
config = (
|
||||
appo.APPOConfig()
|
||||
.environment("CartPole-v1")
|
||||
.env_runners(
|
||||
num_env_runners=1,
|
||||
rollout_fragment_length=10,
|
||||
)
|
||||
.training(
|
||||
train_batch_size_per_learner=20,
|
||||
)
|
||||
.rl_module(
|
||||
model_config=DefaultModelConfig(
|
||||
fcnet_hiddens=[16],
|
||||
vf_share_layers=True,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
algo = config.build()
|
||||
state = algo.get_module(DEFAULT_POLICY_ID).get_state()
|
||||
# Weights and biases for the encoder hidden layer (2) and the output layer
|
||||
# of the policy (2), plus the `log_std_clip` param (1), makes 5 altogether.
|
||||
# We should not get the tensors from the target model here or any value function
|
||||
# related parameters (inference-only).
|
||||
self.assertEqual(len(state), 5)
|
||||
|
||||
def test_env_runner_state_server_on_vs_off(self):
|
||||
"""PULL-based EnvRunnerStateServer: APPO learns with the flag ON and OFF.
|
||||
|
||||
Also checks the global server actor is created only when the flag is enabled.
|
||||
"""
|
||||
for use_server in [False, True]:
|
||||
print(f"Testing with use_server={use_server}")
|
||||
config = (
|
||||
appo.APPOConfig()
|
||||
.environment("CartPole-v1")
|
||||
.env_runners(
|
||||
num_env_runners=2,
|
||||
use_env_runner_state_server=use_server,
|
||||
)
|
||||
)
|
||||
algo = config.build()
|
||||
# The global server actor exists iff the flag is enabled.
|
||||
self.assertEqual(algo._env_runner_state_server is not None, use_server)
|
||||
|
||||
results = algo.train()
|
||||
check_train_results_new_api_stack(results)
|
||||
algo.stop()
|
||||
|
||||
def test_env_runner_state_server_kill_and_recover(self):
|
||||
"""Killing the EnvRunnerStateServer must not stop training; it recovers."""
|
||||
config = (
|
||||
appo.APPOConfig()
|
||||
.environment("CartPole-v1")
|
||||
.env_runners(num_env_runners=2, use_env_runner_state_server=True)
|
||||
)
|
||||
algo = config.build()
|
||||
self.assertIsNotNone(algo._env_runner_state_server)
|
||||
|
||||
for _ in range(3):
|
||||
algo.train()
|
||||
version_before = ray.get(algo._env_runner_state_server.get_version.remote())
|
||||
self.assertGreater(version_before, 0)
|
||||
|
||||
# Kill the server. `max_restarts=-1` makes Ray restart it (with empty state).
|
||||
ray.kill(algo._env_runner_state_server, no_restart=False)
|
||||
|
||||
# Training continues through the gap and the next push re-seeds the server.
|
||||
for _ in range(3):
|
||||
results = algo.train()
|
||||
check_train_results_new_api_stack(results)
|
||||
version_after = ray.get(algo._env_runner_state_server.get_version.remote())
|
||||
self.assertGreaterEqual(version_after, version_before)
|
||||
|
||||
algo.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
@@ -0,0 +1,125 @@
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
import tree # pip install dm_tree
|
||||
|
||||
import ray
|
||||
import ray.rllib.algorithms.appo as appo
|
||||
from ray.rllib.algorithms.appo.appo import LEARNER_RESULTS_CURR_KL_COEFF_KEY
|
||||
from ray.rllib.core import DEFAULT_MODULE_ID
|
||||
from ray.rllib.core.columns import Columns
|
||||
from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig
|
||||
from ray.rllib.policy.sample_batch import SampleBatch
|
||||
from ray.rllib.utils.metrics import LEARNER_RESULTS
|
||||
from ray.rllib.utils.torch_utils import convert_to_torch_tensor
|
||||
|
||||
frag_length = 50
|
||||
|
||||
FAKE_BATCH = {
|
||||
Columns.OBS: np.random.uniform(low=0, high=1, size=(frag_length, 4)).astype(
|
||||
np.float32
|
||||
),
|
||||
Columns.ACTIONS: np.random.choice(2, frag_length).astype(np.float32),
|
||||
Columns.REWARDS: np.random.uniform(low=-1, high=1, size=(frag_length,)).astype(
|
||||
np.float32
|
||||
),
|
||||
Columns.TERMINATEDS: np.array(
|
||||
[False for _ in range(frag_length - 1)] + [True]
|
||||
).astype(np.float32),
|
||||
Columns.VF_PREDS: np.array(list(reversed(range(frag_length))), dtype=np.float32),
|
||||
Columns.ACTION_LOGP: np.log(
|
||||
np.random.uniform(low=0, high=1, size=(frag_length,))
|
||||
).astype(np.float32),
|
||||
Columns.LOSS_MASK: np.ones(shape=(frag_length,)),
|
||||
}
|
||||
|
||||
|
||||
class TestAPPOLearner(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
ray.init()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
ray.shutdown()
|
||||
|
||||
def test_appo_loss(self):
|
||||
"""Test that appo_policy_rlm loss matches the appo learner loss."""
|
||||
config = (
|
||||
appo.APPOConfig()
|
||||
.environment("CartPole-v1")
|
||||
.env_runners(
|
||||
num_env_runners=0,
|
||||
rollout_fragment_length=frag_length,
|
||||
)
|
||||
.training(
|
||||
gamma=0.99,
|
||||
model=dict(
|
||||
fcnet_hiddens=[10, 10],
|
||||
fcnet_activation="linear",
|
||||
vf_share_layers=False,
|
||||
),
|
||||
)
|
||||
)
|
||||
# We have to set exploration_config here manually because setting it through
|
||||
# config.env_runners() only deep-updates it
|
||||
config.exploration_config = {}
|
||||
|
||||
algo = config.build()
|
||||
|
||||
train_batch = SampleBatch(
|
||||
tree.map_structure(lambda x: convert_to_torch_tensor(x), FAKE_BATCH)
|
||||
)
|
||||
|
||||
algo_config = config.copy(copy_frozen=False)
|
||||
algo_config.learners(num_learners=0).experimental(_validate_config=False)
|
||||
algo_config.validate()
|
||||
|
||||
learner_group = algo_config.build_learner_group(env=algo.env_runner.env)
|
||||
learner_group.update(batch=train_batch.as_multi_agent())
|
||||
|
||||
algo.stop()
|
||||
|
||||
def test_kl_coeff_changes(self):
|
||||
initial_kl_coeff = 0.01
|
||||
config = (
|
||||
appo.APPOConfig()
|
||||
.environment("CartPole-v1")
|
||||
.env_runners(
|
||||
num_env_runners=0,
|
||||
rollout_fragment_length=frag_length,
|
||||
exploration_config={},
|
||||
)
|
||||
.learners(num_learners=0)
|
||||
.experimental(_validate_config=False)
|
||||
.training(
|
||||
use_kl_loss=True,
|
||||
kl_coeff=initial_kl_coeff,
|
||||
)
|
||||
.rl_module(
|
||||
model_config=DefaultModelConfig(
|
||||
fcnet_hiddens=[10, 10],
|
||||
fcnet_activation="linear",
|
||||
vf_share_layers=False,
|
||||
),
|
||||
)
|
||||
)
|
||||
algo = config.build()
|
||||
# Call train while results aren't returned because this is
|
||||
# a asynchronous algorithm and results are returned asynchronously.
|
||||
curr_kl_coeff = None
|
||||
while curr_kl_coeff is None:
|
||||
results = algo.train()
|
||||
print(results)
|
||||
results = results.get(LEARNER_RESULTS, {})
|
||||
results = results.get(DEFAULT_MODULE_ID, {})
|
||||
curr_kl_coeff = results.get(LEARNER_RESULTS_CURR_KL_COEFF_KEY)
|
||||
self.assertNotEqual(curr_kl_coeff, initial_kl_coeff)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Test how APPO handles per-policy data imbalance in multi-agent setups.
|
||||
|
||||
Note: PPO will always use "equalize" data across policies. So each policy will train on the same amount of data.
|
||||
APPO, in contrast to PPO, will train on varying amounts of data per policy.
|
||||
|
||||
When a policy_mapping_fn maps more agents to one policy than another, the resulting
|
||||
MultiAgentBatch has unequal per-policy data. This test verifies:
|
||||
1. Default APPO (no minibatch_size): policies train on unequal amounts of data.
|
||||
2. With minibatch_size set: MiniBatchCyclicIterator equalizes per-policy batch sizes.
|
||||
"""
|
||||
|
||||
from ray.rllib.algorithms.appo import APPOConfig
|
||||
from ray.rllib.examples.envs.classes.multi_agent import MultiAgentCartPole
|
||||
from ray.rllib.utils.metrics import NUM_MODULE_STEPS_TRAINED
|
||||
|
||||
NUM_AGENTS = 5
|
||||
|
||||
|
||||
def policy_mapping_fn(agent_id, episode, **kw):
|
||||
return "policy_a" if agent_id in (0, 1, 2, 3) else "policy_b"
|
||||
|
||||
|
||||
def _build_config(*, minibatch_size=None, num_epochs=1):
|
||||
config = (
|
||||
APPOConfig()
|
||||
.environment(MultiAgentCartPole, env_config={"num_agents": NUM_AGENTS})
|
||||
.multi_agent(
|
||||
policies={"policy_a", "policy_b"},
|
||||
policy_mapping_fn=policy_mapping_fn,
|
||||
)
|
||||
.training(
|
||||
train_batch_size_per_learner=500,
|
||||
)
|
||||
.learners(num_learners=0)
|
||||
.env_runners(num_env_runners=1)
|
||||
)
|
||||
if minibatch_size is not None:
|
||||
config.training(minibatch_size=minibatch_size, num_epochs=num_epochs)
|
||||
return config
|
||||
|
||||
|
||||
def test_default_appo_unequal_data():
|
||||
"""Without minibatch_size, policy_a trains on more data than policy_b."""
|
||||
algo = _build_config().build()
|
||||
|
||||
learners = algo.train()["learners"]
|
||||
steps_a = learners["policy_a"][NUM_MODULE_STEPS_TRAINED]
|
||||
steps_b = learners["policy_b"][NUM_MODULE_STEPS_TRAINED]
|
||||
# steps_a should be 4x more data than steps_b
|
||||
assert steps_a / steps_b > 2.5, (
|
||||
"Expected policy_a to train on more data than policy_b "
|
||||
"with biased policy mapping and no minibatch_size."
|
||||
)
|
||||
|
||||
|
||||
def test_minibatch_equalizes_data():
|
||||
"""With minibatch_size, both policies train on equal amounts of data."""
|
||||
algo = _build_config(minibatch_size=50, num_epochs=4).build()
|
||||
|
||||
learners = algo.train()["learners"]
|
||||
steps_a = learners["policy_a"][NUM_MODULE_STEPS_TRAINED]
|
||||
steps_b = learners["policy_b"][NUM_MODULE_STEPS_TRAINED]
|
||||
assert steps_a == steps_b, (
|
||||
"Expected equal per-policy training steps when "
|
||||
"minibatch_size is set (MiniBatchCyclicIterator)."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
Reference in New Issue
Block a user