Custom Model encode attribute error

RLlib
1
  • High: It blocks me to complete my task.

I have this custom model which process two observations, one with CNN and one with dense

class CustomModel(TorchModelV2, nn.Module):
    def __init__(self, obs_space, action_space, num_outputs, model_config, name):
        TorchModelV2.__init__(
            self, obs_space, action_space, num_outputs, model_config, name
        )
        nn.Module.__init__(self)
        self.observation_space = obs_space
        self.action_space = action_space
        
        self.conv = nn.Conv2d(1, 32, kernel_size=3, stride=1)
        self.fc1 = nn.Linear(71*30*32, 256)
        self.fc2 = nn.Linear(256 + 12, 64)
        self.fc3 = nn.Linear(64, action_space.shape[0])
        
    def forward(self, observation, state, mask):
        obs1 = observation["obs1"]
        obs2 = observation["obs2"]
        
        obs1 = obs1.unsqueeze(0).unsqueeze(0)
        obs1 = F.relu(self.conv(obs1))
        obs1 = obs1.view(-1, 71*30*32)
        obs1 = F.relu(self.fc1(obs1))
        
        x = torch.cat([obs1, obs2], dim=1)
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        
        return x, state

And I get this error about encode attribute:

Traceback (most recent call last):
  File "D:\TestProj\kub_IMPALA.py", line 234, in <module>
    trainer = ppo.APPOTrainer(config=cfg)
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\appo\appo.py", line 201, in __init__
    super().__init__(config, *args, **kwargs)
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\algorithm.py", line 441, in __init__
    super().__init__(
  File "D:\TestProj\.env\lib\site-packages\ray\tune\trainable\trainable.py", line 169, in __init__  
    self.setup(copy.deepcopy(self.config))
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\appo\appo.py", line 210, in setup   
    super().setup(config)
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\impala\impala.py", line 464, in setup
    super().setup(config)
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\algorithm.py", line 566, in setup   
    self.workers = WorkerSet(
  File "D:\TestProj\.env\lib\site-packages\ray\rllib\evaluation\worker_set.py", line 191, in __init__
    raise e.args[0].args[2]
AttributeError: 'CustomModel' object has no attribute 'encode'
(RolloutWorker pid=558772) 2023-02-07 22:17:11,837      ERROR worker.py:763 -- Exception raised in creation task: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=558772, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x0000019A9D35CAC0>)
(RolloutWorker pid=558772)   File "python\ray\_raylet.pyx", line 830, in ray._raylet.execute_task
(RolloutWorker pid=558772)   File "python\ray\_raylet.pyx", line 834, in ray._raylet.execute_task
(RolloutWorker pid=558772)   File "python\ray\_raylet.pyx", line 780, in ray._raylet.execute_task.function_executor        
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\_private\function_manager.py", line 674, in actor_method_executor
(RolloutWorker pid=558772)     return method(__ray_actor, *args, **kwargs)
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\util\tracing\tracing_helper.py", line 466, in _resume_span
(RolloutWorker pid=558772)     return method(self, *_args, **_kwargs)
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 712, in __init__
(RolloutWorker pid=558772)     self._build_policy_map(
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\util\tracing\tracing_helper.py", line 466, in _resume_span
(RolloutWorker pid=558772)     return method(self, *_args, **_kwargs)
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 1970, in _build_policy_map
(RolloutWorker pid=558772)     self.policy_map.create_policy(
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\policy\policy_map.py", line 146, in create_policy
(RolloutWorker pid=558772)     policy = create_policy_for_framework(
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\utils\policy.py", line 126, in create_policy_for_framework
(RolloutWorker pid=558772)     return policy_class(observation_space, action_space, merged_config)
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\algorithms\appo\appo_torch_policy.py", line 78, in __init__
(RolloutWorker pid=558772)     TorchPolicyV2.__init__(
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\policy\torch_policy_v2.py", line 88, in __init__
(RolloutWorker pid=558772)     model, dist_class = self._init_model_and_dist_class()
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\rllib\policy\torch_policy_v2.py", line 446, in _init_model_and_dist_class
(RolloutWorker pid=558772)     value = _internal_kv_get(_make_key(self._prefix, category, key))
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\ray\tune\registry.py", line 166, in _make_key
(RolloutWorker pid=558772)     + key.encode("ascii")
(RolloutWorker pid=558772)   File "D:\TestProj\.env\lib\site-packages\torch\nn\modules\module.py", line 1269, in __getattr__
(RolloutWorker pid=558772)     raise AttributeError("'{}' object has no attribute '{}'".format(
(RolloutWorker pid=558772) AttributeError: 'CustomModel' object has no attribute 'encode'

this is the initialization

    env = MyEnv()
    policy_network = CustomModel(
        obs_space=env.observation_space,
        action_space=env.action_space,
        num_outputs=env.action_space.shape[0],
        model_config={},
        name="CustomModel"
    )
2

Hi @evo11x,

Can you share the config you are using?

1 Like
3

@mannyv this is the config

cfg = ppo.appo.DEFAULT_CONFIG.copy()
cfg_custom = {
        "env": MyEnv,
        "num_gpus": 1,
        "num_workers": 5,
        "num_envs_per_worker": 15,
        "framework": "torch",
        "horizon": 500,
        "lr": 0.0000001, # 0.000005        
        "momentum": 0.9,        
        "num_sgd_iter": 30,
        "train_batch_size": 20000,        
        "model": {
          "custom_model": policy_network
        },        
        "evaluation_duration_unit": "episodes",
        "evaluation_num_workers": 1,
        "evaluation_interval": 50,        
    }    
    cfg.update(cfg_custom)
4

And this is the observation

self.observation_space = gym.spaces.Dict({          
            "obs1": gym.spaces.Box(low=0.0, high=255.0, shape=(71,30), dtype=np.float32),
            "obs2": gym.spaces.Box(low=-np.inf, high=np.inf, shape=(12,), dtype=np.float32)            
        })
5

Hi @evo11x,

I think it should be:

model": {
          "custom_model": CustomModel
        },
2 Likes
6

Thanks!, now I get this error

in forward
    obs1 = observation['obs1']
  File "d:\TestProj\.env\lib\site-packages\ray\rllib\policy\sample_batch.py", line 818, in __getitem__
    value = dict.__getitem__(self, key)
KeyError: 'obs1'

the observation key is ok in the environment, but I don’t know how to access it from the custom model

7

Ok, I found how to access the obs1,
observation[‘obs’][‘obs1’]

but there is also an item ‘obs_new’ in observation which I don’t know what it is

8

Do you mean new_obs?

This would be the during the training steps but not the rollout steps.

2 Likes