RLLIB Custom_keras_model

Hi i run this custom_keras_model.py example code with
os: ubuntu 18.04
ray 1.13

here you rllib github code:
“”“Example of using a custom ModelV2 Keras-style model.”“”

import argparse
import os

import ray
from ray import tune
from ray.rllib.agents.dqn.distributional_q_tf_model import DistributionalQTFModel
from ray.rllib.models import ModelCatalog
from ray.rllib.models.tf.misc import normc_initializer
from ray.rllib.models.tf.tf_modelv2 import TFModelV2
from ray.rllib.models.tf.visionnet import VisionNetwork as MyVisionNetwork
from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
from ray.rllib.utils.framework import try_import_tf
from ray.rllib.utils.metrics.learner_info import LEARNER_INFO, LEARNER_STATS_KEY

tf1, tf, tfv = try_import_tf()

parser = argparse.ArgumentParser()
parser.add_argument(
“–run”, type=str, default=“DQN”, help=“The RLlib-registered algorithm to use.”
)
parser.add_argument(“–stop”, type=int, default=200)
parser.add_argument(“–use-vision-network”, action=“store_true”)
parser.add_argument(“–num-cpus”, type=int, default=0)

class MyKerasModel(TFModelV2):
“”“Custom model for policy gradient algorithms.”“”

def __init__(self, obs_space, action_space, num_outputs, model_config, name):
    super(MyKerasModel, self).__init__(
        obs_space, action_space, num_outputs, model_config, name
    )
    self.inputs = tf.keras.layers.Input(shape=obs_space.shape, name="observations")
    layer_1 = tf.keras.layers.Dense(
        128,
        name="my_layer1",
        activation=tf.nn.relu,
        kernel_initializer=normc_initializer(1.0),
    )(self.inputs)
    layer_out = tf.keras.layers.Dense(
        num_outputs,
        name="my_out",
        activation=None,
        kernel_initializer=normc_initializer(0.01),
    )(layer_1)
    value_out = tf.keras.layers.Dense(
        1,
        name="value_out",
        activation=None,
        kernel_initializer=normc_initializer(0.01),
    )(layer_1)
    self.base_model = tf.keras.Model(self.inputs, [layer_out, value_out])

def forward(self, input_dict, state, seq_lens):
    model_out, self._value_out = self.base_model(input_dict["obs"])
    return model_out, state

def value_function(self):
    return tf.reshape(self._value_out, [-1])

def metrics(self):
    return {"foo": tf.constant(42.0)}

class MyKerasQModel(DistributionalQTFModel):
“”“Custom model for DQN.”“”

def __init__(self, obs_space, action_space, num_outputs, model_config, name, **kw):
    super(MyKerasQModel, self).__init__(
        obs_space, action_space, num_outputs, model_config, name, **kw
    )

    # Define the core model layers which will be used by the other
    # output heads of DistributionalQModel
    self.inputs = tf.keras.layers.Input(shape=obs_space.shape, name="observations")
    layer_1 = tf.keras.layers.Dense(
        128,
        name="my_layer1",
        activation=tf.nn.relu,
        kernel_initializer=normc_initializer(1.0),
    )(self.inputs)
    layer_out = tf.keras.layers.Dense(
        num_outputs,
        name="my_out",
        activation=tf.nn.relu,
        kernel_initializer=normc_initializer(1.0),
    )(layer_1)
    self.base_model = tf.keras.Model(self.inputs, layer_out)

# Implement the core forward method.
def forward(self, input_dict, state, seq_lens):
    model_out = self.base_model(input_dict["obs"])
    return model_out, state

def metrics(self):
    return {"foo": tf.constant(42.0)}

if name == “main”:
args = parser.parse_args()
ray.init(num_cpus=args.num_cpus or None)
ModelCatalog.register_custom_model(
“keras_model”, MyVisionNetwork if args.use_vision_network else MyKerasModel
)
ModelCatalog.register_custom_model(
“keras_q_model”, MyVisionNetwork if args.use_vision_network else MyKerasQModel
)

# Tests https://github.com/ray-project/ray/issues/7293
def check_has_custom_metric(result):
    r = result["result"]["info"][LEARNER_INFO]
    if DEFAULT_POLICY_ID in r:
        r = r[DEFAULT_POLICY_ID].get(LEARNER_STATS_KEY, r[DEFAULT_POLICY_ID])
    assert r["model"]["foo"] == 42, result

if args.run == "DQN":
    extra_config = {"learning_starts": 0}
else:
    extra_config = {}

tune.run(
    args.run,
    stop={"episode_reward_mean": args.stop},
    config=dict(
        extra_config,
        **{
            "env": "BreakoutNoFrameskip-v4"
            if args.use_vision_network
            else "CartPole-v0",
            # Use GPUs iff `RLLIB_NUM_GPUS` env var set to > 0.
            "num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", "0")),
            "callbacks": {
                "on_train_result": check_has_custom_metric,
            },
            "model": {
                "custom_model": "keras_q_model"
                if args.run == "DQN"
                else "keras_model"
            },
            "framework": "tf",
        }
    ),
)

I get this error:
(gym) sb@DES:~/Lab$ python3 custom_keras.py --run PPO --num-cpus 6
2022-07-24 21:38:26,178 INFO services.py:1476 – View the Ray dashboard at http://127.0.0.1:8265
(PPOTrainer pid=14368) 2022-07-24 21:38:33,018 INFO trainer.py:2333 – Your framework setting is ‘tf’, meaning you are using static-graph mode. Set framework=‘tf2’ to enable eager execution with tf2.x. You may also then want to set eager_tracing=True in order to reach similar execution speed as with static-graph mode.
(PPOTrainer pid=14368) 2022-07-24 21:38:33,018 INFO ppo.py:415 – In multi-agent mode, policies will be optimized sequentially by the multi-GPU optimizer. Consider setting simple_optimizer=True if this doesn’t work for you.
(PPOTrainer pid=14368) 2022-07-24 21:38:33,019 WARNING deprecation.py:47 – DeprecationWarning: callbacks dict interface has been deprecated. Use a class extending rllib.agents.callbacks.DefaultCallbacks instead. This will raise an error in the future!
(PPOTrainer pid=14368) 2022-07-24 21:38:33,019 INFO trainer.py:906 – Current log_level is WARN. For more information, set ‘log_level’: ‘INFO’ / ‘DEBUG’ or use the -v and -vv flags.
(RolloutWorker pid=14440) 2022-07-24 21:38:37,191 WARNING deprecation.py:47 – DeprecationWarning: callbacks dict interface has been deprecated. Use a class extending rllib.agents.callbacks.DefaultCallbacks instead. This will raise an error in the future!
(PPOTrainer pid=14368) 2022-07-24 21:38:37,257 ERROR worker.py:451 – Exception raised in creation task: The actor died because of an error raised in its creation task, ray::PPOTrainer.init() (pid=14368, ip=172.27.130.120, repr=PPOTrainer)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/agents/trainer.py”, line 1074, in _init
(PPOTrainer pid=14368) raise NotImplementedError
(PPOTrainer pid=14368) NotImplementedError
(PPOTrainer pid=14368)
(PPOTrainer pid=14368) During handling of the above exception, another exception occurred:
(PPOTrainer pid=14368)
(PPOTrainer pid=14368) ray::PPOTrainer.init() (pid=14368, ip=172.27.130.120, repr=PPOTrainer)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/agents/trainer.py”, line 871, in init
(PPOTrainer pid=14368) config, logger_creator, remote_checkpoint_dir, sync_function_tpl
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/tune/trainable.py”, line 156, in init
(PPOTrainer pid=14368) self.setup(copy.deepcopy(self.config))
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/agents/trainer.py”, line 957, in setup
(PPOTrainer pid=14368) logdir=self.logdir,
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/evaluation/worker_set.py”, line 144, in init
(PPOTrainer pid=14368) lambda p, pid: (pid, p.observation_space, p.action_space)
(PPOTrainer pid=14368) ray.exceptions.RayActorError: The actor died because of an error raised in its creation task, ray::RolloutWorker.init() (pid=14440, ip=172.27.130.120, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f64865492e8>)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/evaluation/rollout_worker.py”, line 634, in init
(PPOTrainer pid=14368) seed=seed,
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/evaluation/rollout_worker.py”, line 1789, in _build_policy_map
(PPOTrainer pid=14368) name, orig_cls, obs_space, act_space, conf, merged_conf
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/policy/policy_map.py”, line 141, in create_policy
(PPOTrainer pid=14368) observation_space, action_space, merged_config
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/policy/tf_policy_template.py”, line 270, in init
(PPOTrainer pid=14368) get_batch_divisibility_req=get_batch_divisibility_req,
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/policy/dynamic_tf_policy.py”, line 211, in init
(PPOTrainer pid=14368) framework=“tf”,
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/models/catalog.py”, line 541, in get_model_v2
(PPOTrainer pid=14368) **customized_model_kwargs,
(PPOTrainer pid=14368) File “custom_keras.py”, line 41, in init
(PPOTrainer pid=14368) )(self.inputs)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py”, line 824, in call
(PPOTrainer pid=14368) self._maybe_build(inputs)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py”, line 2146, in _maybe_build
(PPOTrainer pid=14368) self.build(input_shapes)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/layers/core.py”, line 1021, in build
(PPOTrainer pid=14368) trainable=True)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py”, line 529, in add_weight
(PPOTrainer pid=14368) aggregation=aggregation)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/training/tracking/base.py”, line 712, in _add_variable_with_custom_getter
(PPOTrainer pid=14368) **kwargs_for_getter)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py”, line 139, in make_variable
(PPOTrainer pid=14368) shape=variable_shape if variable_shape else None)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/ops/variables.py”, line 258, in call
(PPOTrainer pid=14368) return cls._variable_v1_call(*args, **kwargs)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/ops/variables.py”, line 219, in _variable_v1_call
(PPOTrainer pid=14368) shape=shape)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/ops/variables.py”, line 65, in getter
(PPOTrainer pid=14368) return captured_getter(captured_previous, **kwargs)
(PPOTrainer pid=14368) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/models/catalog.py”, line 518, in track_var_creation
(PPOTrainer pid=14368) created.add(v.ref())
(PPOTrainer pid=14368) AttributeError: ‘ResourceVariable’ object has no attribute ‘ref’
(RolloutWorker pid=14440) WARNING:tensorflow:From /home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.init (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
(RolloutWorker pid=14440) Instructions for updating:
(RolloutWorker pid=14440) If using Keras pass *_constraint arguments to layers.
(RolloutWorker pid=14440) 2022-07-24 21:38:37,243 ERROR worker.py:451 – Exception raised in creation task: The actor died because of an error raised in its creation task, ray::RolloutWorker.init() (pid=14440, ip=172.27.130.120, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f64865492e8>)
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/evaluation/rollout_worker.py”, line 634, in init
(RolloutWorker pid=14440) seed=seed,
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/evaluation/rollout_worker.py”, line 1789, in _build_policy_map
(RolloutWorker pid=14440) name, orig_cls, obs_space, act_space, conf, merged_conf
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/policy/policy_map.py”, line 141, in create_policy
(RolloutWorker pid=14440) observation_space, action_space, merged_config
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/policy/tf_policy_template.py”, line 270, in init
(RolloutWorker pid=14440) get_batch_divisibility_req=get_batch_divisibility_req,
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/policy/dynamic_tf_policy.py”, line 211, in init
(RolloutWorker pid=14440) framework=“tf”,
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/models/catalog.py”, line 541, in get_model_v2
(RolloutWorker pid=14440) **customized_model_kwargs,
(RolloutWorker pid=14440) File “custom_keras.py”, line 41, in init
(RolloutWorker pid=14440) )(self.inputs)
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py”, line 824, in call
(RolloutWorker pid=14440) self._maybe_build(inputs)
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py”, line 2146, in _maybe_build
(RolloutWorker pid=14440) self.build(input_shapes)
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/layers/core.py”, line 1021, in build
(RolloutWorker pid=14440) trainable=True)
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py”, line 529, in add_weight
(RolloutWorker pid=14440) aggregation=aggregation)
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/training/tracking/base.py”, line 712, in _add_variable_with_custom_getter
(RolloutWorker pid=14440) **kwargs_for_getter)
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py”, line 139, in make_variable
(RolloutWorker pid=14440) shape=variable_shape if variable_shape else None)
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/ops/variables.py”, line 258, in call
(RolloutWorker pid=14440) return cls._variable_v1_call(*args, **kwargs)
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/ops/variables.py”, line 219, in _variable_v1_call
(RolloutWorker pid=14440) shape=shape)
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/ops/variables.py”, line 65, in getter
(RolloutWorker pid=14440) return captured_getter(captured_previous, **kwargs)
(RolloutWorker pid=14440) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/models/catalog.py”, line 518, in track_var_creation
(RolloutWorker pid=14440) created.add(v.ref())
(RolloutWorker pid=14440) AttributeError: ‘ResourceVariable’ object has no attribute ‘ref’
(RolloutWorker pid=14441) 2022-07-24 21:38:37,237 WARNING deprecation.py:47 – DeprecationWarning: callbacks dict interface has been deprecated. Use a class extending rllib.agents.callbacks.DefaultCallbacks instead. This will raise an error in the future!
(RolloutWorker pid=14441) WARNING:tensorflow:From /home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.init (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
(RolloutWorker pid=14441) Instructions for updating:
(RolloutWorker pid=14441) If using Keras pass *_constraint arguments to layers.
(RolloutWorker pid=14441) 2022-07-24 21:38:37,301 ERROR worker.py:451 – Exception raised in creation task: The actor died because of an error raised in its creation task, ray::RolloutWorker.init() (pid=14441, ip=172.27.130.120, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f2bf82ce2e8>)
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/evaluation/rollout_worker.py”, line 634, in init
(RolloutWorker pid=14441) seed=seed,
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/evaluation/rollout_worker.py”, line 1789, in _build_policy_map
(RolloutWorker pid=14441) name, orig_cls, obs_space, act_space, conf, merged_conf
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/policy/policy_map.py”, line 141, in create_policy
(RolloutWorker pid=14441) observation_space, action_space, merged_config
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/policy/tf_policy_template.py”, line 270, in init
(RolloutWorker pid=14441) get_batch_divisibility_req=get_batch_divisibility_req,
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/policy/dynamic_tf_policy.py”, line 211, in init
(RolloutWorker pid=14441) framework=“tf”,
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/models/catalog.py”, line 541, in get_model_v2
(RolloutWorker pid=14441) **customized_model_kwargs,
(RolloutWorker pid=14441) File “custom_keras.py”, line 41, in init
(RolloutWorker pid=14441) )(self.inputs)
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py”, line 824, in call
(RolloutWorker pid=14441) self._maybe_build(inputs)
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py”, line 2146, in _maybe_build
(RolloutWorker pid=14441) self.build(input_shapes)
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/layers/core.py”, line 1021, in build
(RolloutWorker pid=14441) trainable=True)
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py”, line 529, in add_weight
(RolloutWorker pid=14441) aggregation=aggregation)
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/training/tracking/base.py”, line 712, in _add_variable_with_custom_getter
(RolloutWorker pid=14441) **kwargs_for_getter)
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py”, line 139, in make_variable
(RolloutWorker pid=14441) shape=variable_shape if variable_shape else None)
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/ops/variables.py”, line 258, in call
(RolloutWorker pid=14441) return cls._variable_v1_call(*args, **kwargs)
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/ops/variables.py”, line 219, in _variable_v1_call
(RolloutWorker pid=14441) shape=shape)
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/tensorflow_core/python/ops/variables.py”, line 65, in getter
(RolloutWorker pid=14441) return captured_getter(captured_previous, **kwargs)
(RolloutWorker pid=14441) File “/home/sb/anaconda3/envs/gym/lib/python3.6/site-packages/ray/rllib/models/catalog.py”, line 518, in track_var_creation
(RolloutWorker pid=14441) created.add(v.ref())
(RolloutWorker pid=14441) AttributeError: ‘ResourceVariable’ object has no attribute ‘ref’

I updated Tensorflow from 1.15 to tf2 now i don’t have this problem!

1 Like