Custom Model encode attribute error

  • High: It blocks me to complete my task.

I have this custom model which process two observations, one with CNN and one with dense

class CustomModel(TorchModelV2, nn.Module):
    def __init__(self, obs_space, action_space, num_outputs, model_config, name):
            self, obs_space, action_space, num_outputs, model_config, name
        self.observation_space = obs_space
        self.action_space = action_space
        self.conv = nn.Conv2d(1, 32, kernel_size=3, stride=1)
        self.fc1 = nn.Linear(71*30*32, 256)
        self.fc2 = nn.Linear(256 + 12, 64)
        self.fc3 = nn.Linear(64, action_space.shape[0])
    def forward(self, observation, state, mask):
        obs1 = observation["obs1"]
        obs2 = observation["obs2"]
        obs1 = obs1.unsqueeze(0).unsqueeze(0)
        obs1 = F.relu(self.conv(obs1))
        obs1 = obs1.view(-1, 71*30*32)
        obs1 = F.relu(self.fc1(obs1))
        x =[obs1, obs2], dim=1)
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x, state

And I get this error about encode attribute:

Traceback (most recent call last):
  File "D:\TestProj\", line 234, in <module>
    trainer = ppo.APPOTrainer(config=cfg)
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\appo\", line 201, in __init__
    super().__init__(config, *args, **kwargs)
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\", line 441, in __init__
  File "D:\TestProj\.env\lib\site-packages\ray\tune\trainable\", line 169, in __init__  
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\appo\", line 210, in setup   
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\impala\", line 464, in setup
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\", line 566, in setup   
    self.workers = WorkerSet(
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\evaluation\", line 191, in __init__
    raise e.args[0].args[2]
AttributeError: 'CustomModel' object has no attribute 'encode'
(RolloutWorker pid=558772) 2023-02-07 22:17:11,837      ERROR -- Exception raised in creation task: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=558772, ip=, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x0000019A9D35CAC0>)
(RolloutWorker pid=558772)   File "python\ray\_raylet.pyx", line 830, in ray._raylet.execute_task
(RolloutWorker pid=558772)   File "python\ray\_raylet.pyx", line 834, in ray._raylet.execute_task
(RolloutWorker pid=558772)   File "python\ray\_raylet.pyx", line 780, in ray._raylet.execute_task.function_executor        
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\_private\", line 674, in actor_method_executor
(RolloutWorker pid=558772)     return method(__ray_actor, *args, **kwargs)
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\util\tracing\", line 466, in _resume_span
(RolloutWorker pid=558772)     return method(self, *_args, **_kwargs)
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\evaluation\", line 712, in __init__
(RolloutWorker pid=558772)     self._build_policy_map(
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\util\tracing\", line 466, in _resume_span
(RolloutWorker pid=558772)     return method(self, *_args, **_kwargs)
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\evaluation\", line 1970, in _build_policy_map
(RolloutWorker pid=558772)     self.policy_map.create_policy(
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\policy\", line 146, in create_policy
(RolloutWorker pid=558772)     policy = create_policy_for_framework(
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\utils\", line 126, in create_policy_for_framework
(RolloutWorker pid=558772)     return policy_class(observation_space, action_space, merged_config)
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\appo\", line 78, in __init__
(RolloutWorker pid=558772)     TorchPolicyV2.__init__(
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\policy\", line 88, in __init__
(RolloutWorker pid=558772)     model, dist_class = self._init_model_and_dist_class()
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\policy\", line 446, in _init_model_and_dist_class
(RolloutWorker pid=558772)     value = _internal_kv_get(_make_key(self._prefix, category, key))
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\tune\", line 166, in _make_key
(RolloutWorker pid=558772)     + key.encode("ascii")
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\torch\nn\modules\", line 1269, in __getattr__
(RolloutWorker pid=558772)     raise AttributeError("'{}' object has no attribute '{}'".format(
(RolloutWorker pid=558772) AttributeError: 'CustomModel' object has no attribute 'encode'

this is the initialization

    env = MyEnv()
    policy_network = CustomModel(

Hi @evo11x,

Can you share the config you are using?

1 Like

@mannyv this is the config

cfg = ppo.appo.DEFAULT_CONFIG.copy()
cfg_custom = {
        "env": MyEnv,
        "num_gpus": 1,
        "num_workers": 5,
        "num_envs_per_worker": 15,
        "framework": "torch",
        "horizon": 500,
        "lr": 0.0000001, # 0.000005        
        "momentum": 0.9,        
        "num_sgd_iter": 30,
        "train_batch_size": 20000,        
        "model": {
          "custom_model": policy_network
        "evaluation_duration_unit": "episodes",
        "evaluation_num_workers": 1,
        "evaluation_interval": 50,        

And this is the observation

self.observation_space = gym.spaces.Dict({          
            "obs1": gym.spaces.Box(low=0.0, high=255.0, shape=(71,30), dtype=np.float32),
            "obs2": gym.spaces.Box(low=-np.inf, high=np.inf, shape=(12,), dtype=np.float32)            

Hi @evo11x,

I think it should be:

model": {
          "custom_model": CustomModel

Thanks!, now I get this error

in forward
    obs1 = observation['obs1']
  File "d:\TestProj\.env\lib\site-packages\ray\rllib\policy\", line 818, in __getitem__
    value = dict.__getitem__(self, key)
KeyError: 'obs1'

the observation key is ok in the environment, but I don’t know how to access it from the custom model

Ok, I found how to access the obs1,

but there is also an item ‘obs_new’ in observation which I don’t know what it is

Do you mean new_obs?

This would be the during the training steps but not the rollout steps.