chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,134 @@
|
||||
# @OldAPIStack
|
||||
|
||||
"""Example of creating a custom input API
|
||||
|
||||
Custom input apis are useful when your data source is in a custom format or
|
||||
when it is necessary to use an external data loading mechanism.
|
||||
In this example, we train an rl agent on user specified input data.
|
||||
Instead of using the built in JsonReader, we will create our own custom input
|
||||
api, and show how to pass config arguments to it.
|
||||
|
||||
To train CQL on the pendulum environment:
|
||||
$ python custom_input_api.py --input-files=../offline/tests/data/pendulum/enormous.zip
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import ray
|
||||
from ray import tune
|
||||
from ray.rllib.offline import InputReader, IOContext, JsonReader, ShuffledInput
|
||||
from ray.rllib.utils.metrics import (
|
||||
ENV_RUNNER_RESULTS,
|
||||
EPISODE_RETURN_MEAN,
|
||||
EVALUATION_RESULTS,
|
||||
)
|
||||
from ray.tune.registry import get_trainable_cls, register_input
|
||||
from ray.tune.result import TRAINING_ITERATION
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--run", type=str, default="CQL", help="The RLlib-registered algorithm to use."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--framework",
|
||||
choices=["tf", "tf2", "torch"],
|
||||
default="torch",
|
||||
help="The DL framework specifier.",
|
||||
)
|
||||
parser.add_argument("--stop-iters", type=int, default=100)
|
||||
parser.add_argument(
|
||||
"--input-files",
|
||||
type=str,
|
||||
default=os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
"../../offline/tests/data/pendulum/small.json",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class CustomJsonReader(JsonReader):
|
||||
"""
|
||||
Example custom InputReader implementation (extended from JsonReader).
|
||||
|
||||
This gets wrapped in ShuffledInput to comply with offline rl algorithms.
|
||||
"""
|
||||
|
||||
def __init__(self, ioctx: IOContext):
|
||||
"""
|
||||
The constructor must take an IOContext to be used in the input config.
|
||||
Args:
|
||||
ioctx: use this to access the `input_config` arguments.
|
||||
"""
|
||||
super().__init__(ioctx.input_config["input_files"], ioctx)
|
||||
|
||||
|
||||
def input_creator(ioctx: IOContext) -> InputReader:
|
||||
"""
|
||||
The input creator method can be used in the input registry or set as the
|
||||
config["input"] parameter.
|
||||
|
||||
Args:
|
||||
ioctx: use this to access the `input_config` arguments.
|
||||
|
||||
Returns:
|
||||
instance of ShuffledInput to work with some offline rl algorithms
|
||||
"""
|
||||
return ShuffledInput(CustomJsonReader(ioctx))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ray.init()
|
||||
args = parser.parse_args()
|
||||
|
||||
# make absolute path because relative path looks in result directory
|
||||
args.input_files = os.path.abspath(args.input_files)
|
||||
|
||||
# we register our custom input creator with this convenient function
|
||||
register_input("custom_input", input_creator)
|
||||
|
||||
# Config modified from rllib/examples/algorithms/cql/pendulum-cql.yaml
|
||||
default_config = get_trainable_cls(args.run).get_default_config()
|
||||
config = (
|
||||
default_config.environment("Pendulum-v1", clip_actions=True)
|
||||
.framework(args.framework)
|
||||
.offline_data(
|
||||
# We can either use the tune registry ...
|
||||
input_="custom_input",
|
||||
# ... full classpath
|
||||
# input_: "ray.rllib.examples.offline_rl.custom_input_api.CustomJsonReader"
|
||||
# ... or a direct function to connect our input api.
|
||||
# input_: input_creator
|
||||
input_config={"input_files": args.input_files}, # <- passed to IOContext
|
||||
actions_in_input_normalized=True,
|
||||
)
|
||||
.training(train_batch_size=2000)
|
||||
.evaluation(
|
||||
evaluation_interval=1,
|
||||
evaluation_num_env_runners=2,
|
||||
evaluation_duration=10,
|
||||
evaluation_parallel_to_training=True,
|
||||
evaluation_config=default_config.overrides(
|
||||
input_="sampler",
|
||||
explore=False,
|
||||
),
|
||||
)
|
||||
.reporting(metrics_num_episodes_for_smoothing=5)
|
||||
)
|
||||
|
||||
if args.run == "CQL":
|
||||
config.training(
|
||||
twin_q=True,
|
||||
num_steps_sampled_before_learning_starts=0,
|
||||
bc_iters=100,
|
||||
)
|
||||
|
||||
stop = {
|
||||
TRAINING_ITERATION: args.stop_iters,
|
||||
f"{EVALUATION_RESULTS}/{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": -600,
|
||||
}
|
||||
|
||||
tuner = tune.Tuner(
|
||||
args.run, param_space=config, run_config=tune.RunConfig(stop=stop, verbose=1)
|
||||
)
|
||||
tuner.fit()
|
||||
@@ -0,0 +1,168 @@
|
||||
# @OldAPIStack
|
||||
|
||||
"""Example on how to use CQL to learn from an offline JSON file.
|
||||
|
||||
Important node: Make sure that your offline data file contains only
|
||||
a single timestep per line to mimic the way SAC pulls samples from
|
||||
the buffer.
|
||||
|
||||
Generate the offline json file by running an SAC algo until it reaches expert
|
||||
level on your command line. For example:
|
||||
$ cd ray
|
||||
$ rllib train -f rllib/examples/algorithms/sac/pendulum-sac.yaml --no-ray-ui
|
||||
|
||||
Also make sure that in the above SAC yaml file (pendulum-sac.yaml),
|
||||
you specify an additional "output" key with any path on your local
|
||||
file system. In that path, the offline json files will be written to.
|
||||
|
||||
Use the generated file(s) as "input" in the CQL config below
|
||||
(`config["input"] = [list of your json files]`), then run this script.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ray.rllib.algorithms import cql as cql
|
||||
from ray.rllib.execution.rollout_ops import (
|
||||
synchronous_parallel_sample,
|
||||
)
|
||||
from ray.rllib.policy.sample_batch import convert_ma_batch_to_sample_batch
|
||||
from ray.rllib.utils.framework import try_import_torch
|
||||
from ray.rllib.utils.metrics import (
|
||||
ENV_RUNNER_RESULTS,
|
||||
EPISODE_RETURN_MEAN,
|
||||
EVALUATION_RESULTS,
|
||||
)
|
||||
|
||||
torch, _ = try_import_torch()
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--as-test",
|
||||
action="store_true",
|
||||
help="Whether this script should be run as a test: --stop-reward must "
|
||||
"be achieved within --stop-timesteps AND --stop-iters.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stop-iters", type=int, default=5, help="Number of iterations to train."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stop-reward", type=float, default=50.0, help="Reward at which we stop training."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parser.parse_args()
|
||||
|
||||
# See rllib/examples/algorithms/cql/pendulum-cql.yaml for comparison.
|
||||
config = (
|
||||
cql.CQLConfig()
|
||||
.api_stack(
|
||||
enable_env_runner_and_connector_v2=False,
|
||||
enable_rl_module_and_learner=False,
|
||||
)
|
||||
.framework(framework="torch")
|
||||
.env_runners(num_env_runners=0)
|
||||
.training(
|
||||
n_step=3,
|
||||
bc_iters=0,
|
||||
clip_actions=False,
|
||||
tau=0.005,
|
||||
target_entropy="auto",
|
||||
q_model_config={
|
||||
"fcnet_hiddens": [256, 256],
|
||||
"fcnet_activation": "relu",
|
||||
},
|
||||
policy_model_config={
|
||||
"fcnet_hiddens": [256, 256],
|
||||
"fcnet_activation": "relu",
|
||||
},
|
||||
optimization_config={
|
||||
"actor_learning_rate": 3e-4,
|
||||
"critic_learning_rate": 3e-4,
|
||||
"entropy_learning_rate": 3e-4,
|
||||
},
|
||||
train_batch_size=256,
|
||||
target_network_update_freq=1,
|
||||
num_steps_sampled_before_learning_starts=256,
|
||||
)
|
||||
.reporting(min_train_timesteps_per_iteration=1000)
|
||||
.debugging(log_level="INFO")
|
||||
.environment("Pendulum-v1", normalize_actions=True)
|
||||
.offline_data(
|
||||
input_config={
|
||||
"paths": ["offline/tests/data/pendulum/enormous.zip"],
|
||||
"format": "json",
|
||||
}
|
||||
)
|
||||
.evaluation(
|
||||
evaluation_num_env_runners=1,
|
||||
evaluation_interval=1,
|
||||
evaluation_duration=10,
|
||||
evaluation_parallel_to_training=False,
|
||||
evaluation_config=cql.CQLConfig.overrides(input_="sampler"),
|
||||
)
|
||||
)
|
||||
# evaluation_parallel_to_training should be False b/c iterations are very long
|
||||
# and this would cause evaluation to lag one iter behind training.
|
||||
|
||||
# Check, whether we can learn from the given file in `num_iterations`
|
||||
# iterations, up to a reward of `min_reward`.
|
||||
num_iterations = 5
|
||||
min_reward = -300
|
||||
|
||||
cql_algorithm = cql.CQL(config=config)
|
||||
learnt = False
|
||||
for i in range(num_iterations):
|
||||
print(f"Iter {i}")
|
||||
eval_results = cql_algorithm.train().get(EVALUATION_RESULTS)
|
||||
if eval_results:
|
||||
print(
|
||||
"... R={}".format(eval_results[ENV_RUNNER_RESULTS][EPISODE_RETURN_MEAN])
|
||||
)
|
||||
# Learn until some reward is reached on an actual live env.
|
||||
if eval_results[ENV_RUNNER_RESULTS][EPISODE_RETURN_MEAN] >= min_reward:
|
||||
# Test passed gracefully.
|
||||
if args.as_test:
|
||||
print("Test passed after {} iterations.".format(i))
|
||||
quit(0)
|
||||
learnt = True
|
||||
break
|
||||
|
||||
# Get policy and model.
|
||||
cql_policy = cql_algorithm.get_policy()
|
||||
cql_model = cql_policy.model
|
||||
|
||||
# If you would like to query CQL's learnt Q-function for arbitrary
|
||||
# (cont.) actions, do the following:
|
||||
obs_batch = torch.from_numpy(np.random.random(size=(5, 3)))
|
||||
action_batch = torch.from_numpy(np.random.random(size=(5, 1)))
|
||||
q_values = cql_model.get_q_values(obs_batch, action_batch)[0]
|
||||
# If you are using the "twin_q", there'll be 2 Q-networks and
|
||||
# we usually consider the min of the 2 outputs, like so:
|
||||
twin_q_values = cql_model.get_twin_q_values(obs_batch, action_batch)[0]
|
||||
final_q_values = torch.min(q_values, twin_q_values)[0]
|
||||
print(f"final_q_values={final_q_values.detach().numpy()}")
|
||||
|
||||
# Example on how to do evaluation on the trained Algorithm.
|
||||
# using the data from our buffer.
|
||||
# Get a sample (MultiAgentBatch).
|
||||
|
||||
batch = synchronous_parallel_sample(worker_set=cql_algorithm.env_runner_group)
|
||||
batch = convert_ma_batch_to_sample_batch(batch)
|
||||
obs = torch.from_numpy(batch["obs"])
|
||||
# Pass the observations through our model to get the
|
||||
# features, which then to pass through the Q-head.
|
||||
model_out, _ = cql_model({"obs": obs})
|
||||
# The estimated Q-values from the (historic) actions in the batch.
|
||||
q_values_old = cql_model.get_q_values(
|
||||
model_out, torch.from_numpy(batch["actions"])
|
||||
)[0]
|
||||
# The estimated Q-values for the new actions computed by our policy.
|
||||
actions_new = cql_policy.compute_actions_from_input_dict({"obs": obs})[0]
|
||||
q_values_new = cql_model.get_q_values(model_out, torch.from_numpy(actions_new))[0]
|
||||
print(f"Q-val batch={q_values_old.detach().numpy()}")
|
||||
print(f"Q-val policy={q_values_new.detach().numpy()}")
|
||||
|
||||
cql_algorithm.stop()
|
||||
@@ -0,0 +1,60 @@
|
||||
# @OldAPIStack
|
||||
|
||||
"""Simple example of writing experiences to a file using JsonWriter."""
|
||||
|
||||
# __sphinx_doc_begin__
|
||||
import os
|
||||
|
||||
import gymnasium as gym
|
||||
import numpy as np
|
||||
|
||||
from ray._common.utils import get_default_ray_temp_dir
|
||||
from ray.rllib.evaluation.sample_batch_builder import SampleBatchBuilder
|
||||
from ray.rllib.models.preprocessors import get_preprocessor
|
||||
from ray.rllib.offline.json_writer import JsonWriter
|
||||
|
||||
if __name__ == "__main__":
|
||||
batch_builder = SampleBatchBuilder() # or MultiAgentSampleBatchBuilder
|
||||
writer = JsonWriter(os.path.join(get_default_ray_temp_dir(), "demo-out"))
|
||||
|
||||
# You normally wouldn't want to manually create sample batches if a
|
||||
# simulator is available, but let's do it anyways for example purposes:
|
||||
env = gym.make("CartPole-v1")
|
||||
|
||||
# RLlib uses preprocessors to implement transforms such as one-hot encoding
|
||||
# and flattening of tuple and dict observations. For CartPole a no-op
|
||||
# preprocessor is used, but this may be relevant for more complex envs.
|
||||
prep = get_preprocessor(env.observation_space)(env.observation_space)
|
||||
print("The preprocessor is", prep)
|
||||
|
||||
for eps_id in range(100):
|
||||
obs, info = env.reset()
|
||||
prev_action = np.zeros_like(env.action_space.sample())
|
||||
prev_reward = 0
|
||||
terminated = truncated = False
|
||||
t = 0
|
||||
while not terminated and not truncated:
|
||||
action = env.action_space.sample()
|
||||
new_obs, rew, terminated, truncated, info = env.step(action)
|
||||
batch_builder.add_values(
|
||||
t=t,
|
||||
eps_id=eps_id,
|
||||
agent_index=0,
|
||||
obs=prep.transform(obs),
|
||||
actions=action,
|
||||
action_prob=1.0, # put the true action probability here
|
||||
action_logp=0.0,
|
||||
rewards=rew,
|
||||
prev_actions=prev_action,
|
||||
prev_rewards=prev_reward,
|
||||
terminateds=terminated,
|
||||
truncateds=truncated,
|
||||
infos=info,
|
||||
new_obs=prep.transform(new_obs),
|
||||
)
|
||||
obs = new_obs
|
||||
prev_action = action
|
||||
prev_reward = rew
|
||||
t += 1
|
||||
writer.write(batch_builder.build_and_reset())
|
||||
# __sphinx_doc_end__
|
||||
Reference in New Issue
Block a user