RayTaskError(AttributeError) : ray::RolloutWorker.par_iter_next()

RLlib
1

Hello everyone
I am tryting to train a PPO trainer on a multi-agent environment.
I am new to RLlib and I tried to define the environment like the MultiAgentArena example.
This is my config:

policies= {
    "policy_" + str(i): (None , game.observation_space , game.action_space["agent_" + str(i)] , {}) for i in range(0,game.num_agents)
}

def policy_mapping_fn (agent_id: str):
    #assert agent_id in [str(i) for i in range(0,5)], f"ERROR: invalid agent id {agent_id}!!!"
    return "policy_" + str(agent_id[len(agent_id)-1])
        
config={
    "env": MultiUnitAuction,  # "my_env" <- if we previously have registered the env with `tune.register_env("[name]", lambda config: [returns env object])`.
    "create_env_on_driver": True,
    "multiagent": {
        "policies": policies,
        "policy_mapping_fn": policy_mapping_fn,
},
}

But I constantly recieve the following error:

RayTaskError(AttributeError)              Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_17784\1339345677.py in <module>
----> 1 results = rllib_trainer.train()
      2 
      3 # Delete the config from the results for clarity.
      4 # Only the stats will remain, then.
      5 del results["config"]

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\tune\trainable.py in train(self)
    312         """
    313         start = time.time()
--> 314         result = self.step()
    315         assert isinstance(result, dict), "step() needs to return a dict."
    316 

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\agents\trainer.py in step(self)
    878                         "continue training without the failed worker, set "
    879                         "`'ignore_worker_failures': True`.")
--> 880                     raise e
    881             # Any other exception.
    882             except Exception as e:

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\agents\trainer.py in step(self)
    865             # Try to train one step.
    866             try:
--> 867                 result = self.step_attempt()
    868             # @ray.remote RolloutWorker failure -> Try to recover,
    869             # if necessary.

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\agents\trainer.py in step_attempt(self)
    918         # No evaluation necessary, just run the next training iteration.
    919         if not evaluate_this_iter:
--> 920             step_results = next(self.train_exec_impl)
    921         # We have to evaluate in this training iteration.
    922         else:

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in __next__(self)
    754     def __next__(self):
    755         self._build_once()
--> 756         return next(self.built_iterator)
    757 
    758     def __str__(self):

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_filter(it)
    841     def filter(self, fn: Callable[[T], bool]) -> "LocalIterator[T]":
    842         def apply_filter(it):
--> 843             for item in it:
    844                 with self._metrics_context():
    845                     if isinstance(item, _NextValueNotReady) or fn(item):

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_filter(it)
    841     def filter(self, fn: Callable[[T], bool]) -> "LocalIterator[T]":
    842         def apply_filter(it):
--> 843             for item in it:
    844                 with self._metrics_context():
    845                     if isinstance(item, _NextValueNotReady) or fn(item):

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_flatten(it)
    874     def flatten(self) -> "LocalIterator[T[0]]":
    875         def apply_flatten(it):
--> 876             for item in it:
    877                 if isinstance(item, _NextValueNotReady):
    878                     yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in base_iterator(timeout)
    469             while active:
    470                 try:
--> 471                     yield ray.get(futures, timeout=timeout)
    472                     futures = [a.par_iter_next.remote() for a in active]
    473                     # Always yield after each round of gets with timeout.

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\_private\client_mode_hook.py in wrapper(*args, **kwargs)
    103             if func.__name__ != "init" or is_client_mode_enabled_by_default:
    104                 return getattr(ray, func.__name__)(*args, **kwargs)
--> 105         return func(*args, **kwargs)
    106 
    107     return wrapper

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\worker.py in get(object_refs, timeout)
   1711                     worker.core_worker.dump_object_store_memory_usage()
   1712                 if isinstance(value, RayTaskError):
-> 1713                     raise value.as_instanceof_cause()
   1714                 else:
   1715                     raise value

RayTaskError(AttributeError): ray::RolloutWorker.par_iter_next() (pid=16836, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x000001CB6E90FCD0>)
  File "python\ray\_raylet.pyx", line 625, in ray._raylet.execute_task
  File "python\ray\_raylet.pyx", line 629, in ray._raylet.execute_task
  File "python\ray\_raylet.pyx", line 578, in ray._raylet.execute_task.function_executor
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\_private\function_manager.py", line 609, in actor_method_executor
    return method(__ray_actor, *args, **kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\tracing\tracing_helper.py", line 451, in _resume_span
    return method(self, *_args, **_kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py", line 1151, in par_iter_next
    return next(self.local_it)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 381, in gen_rollouts
    yield self.sample()
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\tracing\tracing_helper.py", line 451, in _resume_span
    return method(self, *_args, **_kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 757, in sample
    batches = [self.input_reader.next()]
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 103, in next
    batches = [self.get_data()]
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 265, in get_data
    item = next(self._env_runner)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 633, in _env_runner
    _process_observations(
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 848, in _process_observations
    prep_obs = preprocessor.transform(raw_obs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\models\preprocessors.py", line 195, in transform
    self.check_shape(observation)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\models\preprocessors.py", line 67, in check_shape
    if not self._obs_space.contains(observation):
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\gym\spaces\box.py", line 144, in contains
    x.shape == self.shape and np.all(x >= self.low) and np.all(x <= self.high)
AttributeError: 'int' object has no attribute 'shape'
2022-02-19 12:35:19,509	ERROR worker.py:84 -- Unhandled error (suppress with RAY_IGNORE_UNHANDLED_ERRORS=1): ray::RolloutWorker.par_iter_next() (pid=11860, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x0000020A96C31CD0>)
  File "python\ray\_raylet.pyx", line 625, in ray._raylet.execute_task
  File "python\ray\_raylet.pyx", line 629, in ray._raylet.execute_task
  File "python\ray\_raylet.pyx", line 578, in ray._raylet.execute_task.function_executor
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\_private\function_manager.py", line 609, in actor_method_executor
    return method(__ray_actor, *args, **kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\tracing\tracing_helper.py", line 451, in _resume_span
    return method(self, *_args, **_kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py", line 1151, in par_iter_next
    return next(self.local_it)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 381, in gen_rollouts
    yield self.sample()
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\tracing\tracing_helper.py", line 451, in _resume_span
    return method(self, *_args, **_kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 757, in sample
    batches = [self.input_reader.next()]
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 103, in next
    batches = [self.get_data()]
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 265, in get_data
    item = next(self._env_runner)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 633, in _env_runner
    _process_observations(
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 848, in _process_observations
    prep_obs = preprocessor.transform(raw_obs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\models\preprocessors.py", line 195, in transform
    self.check_shape(observation)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\models\preprocessors.py", line 67, in check_shape
    if not self._obs_space.contains(observation):
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\gym\spaces\box.py", line 144, in contains
    x.shape == self.shape and np.all(x >= self.low) and np.all(x <= self.high)
AttributeError: 'int' object has no attribute 'shape'

I tried to find the similar issues here but It didn’t work.
Has anyone had a similar problem before?

Thanks

2

Hi @amohazab ,

without knowing more about what game is, it is hard to say what exactly is the error. However, the error occurs in the gym.spaces.Box class. As there are some spaces passed by you to the policies dictionary, this would be the first place to look for the space definitions game.observation_space and game.action_space.

Also, in ray I cannot find an example of the MultiAgentArena. I would supply some more information in your thread, if you want more people here to take part in the solution.

3

Hi @Lars_Simon_Zehnder
Thanks for the response

The environment is a multi-unit auction. We have 3 items and 5 bidders at each step and each one of them has a prespecified marginal valuation. At each step, they bid some amount between zero and their valuation. The people whose bids are among the highest three will get positive rewards (difference between valuation and bid value) and the rest will get zero.
Following is the envieonment definition:


import gym
#from gym.spaces import Discrete, MultiDiscrete
import numpy as np
import random

from ray.rllib.env.multi_agent_env import MultiAgentEnv


class MultiUnitAuction(MultiAgentEnv):
    def __init__(self , config = None):
        config = config or {}
        self.marginal_valuation_vector = [5 , 10 , 15 , 12 , 7]
    
        #number of agents
        self.num_agents = 5
        #block_size
        self.block_size = 3
        # define the market clearing price
        self.p_c = 0
        self.observation_space = gym.spaces.Box(low=np.array([0 ]), high=np.array([max(self.marginal_valuation_vector)]), dtype=np.float32)
        self.action_space = gym.spaces.Dict({
            "agent_" + str(i) : gym.spaces.Box(low=np.array([0]), high=np.array([self.marginal_valuation_vector[i]]), dtype=np.float32) for i in range(0,self.num_agents)
        })
        
        self.reset()
        
    def reset(self):
        """Returns initial observation of next(!) episode."""

        self.p_c = 0
        return self._get_obs()

    def step(self, action: dict):
        """
        Returns (next observation, rewards, dones, infos) after having taken the given actions.
        
        e.g.
        `action={"agent1": action_for_agent1, "agent2": action_for_agent2}`
        """
        
        bids = []
        for i in range(0 , self.num_agents):
            bids.append(action["agent_" + str(i)])
        
        self.p_c = sorted(bids)[len(bids) - self.block_size]
        
        
        # Get observations (based on new agent positions).
        obs = self._get_obs()

        # calculate rewards
        r = []
        for i in range(0 , self.num_agents):
            if bids[i] < self.p_c:
                r.append(0)
            else:
                r.append(self.marginal_valuation_vector[i] - bids[i])
        
        
        rewards = {
             "agent_" + str(i) : r[i] for i in range(0,self.num_agents)
        }

        
        #set done === True
        is_done = True
        # Generate a `done` dict (per-agent and total).
        dones = {
            "agent_0" : is_done,
            "agent_1" : is_done,
            "agent_2" : is_done,
            "agent_3" : is_done,
            "agent_4" : is_done,
            # special `__all__` key indicates that the episode is done for all agents.
            "__all__": is_done,
        }

        return obs, rewards, dones, {}  # <- info dict (not needed here).

    def _get_obs(self):
        """
        Returns obs dict (agent name to discrete-pos tuple) using each
        agent's current x/y-positions.
        """
        return {
            "agent_" + str(i) : self.p_c for i in range(0,self.num_agents)
        }

    def render(self, mode=None):
        pass

And I got the MultiAgentArena environment definition from one of the RLlib events before (Ray summit, 2021).

And for the error, I just tried to define the observation space and action space as continuous variables. I’m not sure where I have done wrong. I checked the environment and it is working ok. Is it because of the policy definitions?
Thanks

4

Hi @amohazab ,

spaces are from a gym view correctly implemented. Where it gves you the error is when the <your_space>.contains(observation) is called. The reason is that the Box space you defined for the observations contains arrays by definition in gym, the observation however you put in is of type int. This type of course does not possess a shape attribute and an error occurs.

What lets me wonder so far is that the contains() function in the gym Box space converts the input into an np.ndarray which then by definition should have a shape attribute.

I would debug your program and set a breakpoint at this line in the code to see what observation arrives there.

Good luck!

5

Hi @Lars_Simon_Zehnder

Thanks again for the response.
How can I make an instance of Preprocessor class from here?
I am using:

Preprocessor(gym.spaces.Box(low=np.array([0 ]), high=np.array([10]), dtype=np.float32))

But I get the following error:

NotImplementedError                       Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_4240\216486321.py in <module>
----> 1 game = Preprocessor(gym.spaces.Box(low=np.array([0 ]), high=np.array([10]), dtype=np.float32))

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\models\preprocessors.py in __init__(self, obs_space, options)
     38         else:
     39             self._options = options
---> 40         self.shape = self._init_shape(obs_space, self._options)
     41         self._size = int(np.product(self.shape))
     42         self._i = 0

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\models\preprocessors.py in _init_shape(self, obs_space, options)
     46     def _init_shape(self, obs_space: gym.Space, options: dict) -> List[int]:
     47         """Returns the shape after preprocessing."""
---> 48         raise NotImplementedError
     49 
     50     @PublicAPI

NotImplementedError:

Should I use a gym space as an input for the Preprocessor?

6

@amohazab

Try returning a list instead of an int.

return {
            "agent_" + str(i) : [self.p_c] for i in range(0,self.num_agents)
        }
1 Like
7

Hi @mannyv
Thanks for the response
I tried to do the exact same thing but I received the following error:

RayTaskError(ValueError)                  Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_17616\1339345677.py in <module>
----> 1 results = rllib_trainer.train()
      2 
      3 # Delete the config from the results for clarity.
      4 # Only the stats will remain, then.
      5 del results["config"]

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\tune\trainable.py in train(self)
    312         """
    313         start = time.time()
--> 314         result = self.step()
    315         assert isinstance(result, dict), "step() needs to return a dict."
    316 

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\agents\trainer.py in step(self)
    878                         "continue training without the failed worker, set "
    879                         "`'ignore_worker_failures': True`.")
--> 880                     raise e
    881             # Any other exception.
    882             except Exception as e:

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\agents\trainer.py in step(self)
    865             # Try to train one step.
    866             try:
--> 867                 result = self.step_attempt()
    868             # @ray.remote RolloutWorker failure -> Try to recover,
    869             # if necessary.

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\agents\trainer.py in step_attempt(self)
    918         # No evaluation necessary, just run the next training iteration.
    919         if not evaluate_this_iter:
--> 920             step_results = next(self.train_exec_impl)
    921         # We have to evaluate in this training iteration.
    922         else:

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in __next__(self)
    754     def __next__(self):
    755         self._build_once()
--> 756         return next(self.built_iterator)
    757 
    758     def __str__(self):

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_filter(it)
    841     def filter(self, fn: Callable[[T], bool]) -> "LocalIterator[T]":
    842         def apply_filter(it):
--> 843             for item in it:
    844                 with self._metrics_context():
    845                     if isinstance(item, _NextValueNotReady) or fn(item):

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_filter(it)
    841     def filter(self, fn: Callable[[T], bool]) -> "LocalIterator[T]":
    842         def apply_filter(it):
--> 843             for item in it:
    844                 with self._metrics_context():
    845                     if isinstance(item, _NextValueNotReady) or fn(item):

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_flatten(it)
    874     def flatten(self) -> "LocalIterator[T[0]]":
    875         def apply_flatten(it):
--> 876             for item in it:
    877                 if isinstance(item, _NextValueNotReady):
    878                     yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in apply_foreach(it)
    781 
    782             def apply_foreach(it):
--> 783                 for item in it:
    784                     if isinstance(item, _NextValueNotReady):
    785                         yield item

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py in base_iterator(timeout)
    469             while active:
    470                 try:
--> 471                     yield ray.get(futures, timeout=timeout)
    472                     futures = [a.par_iter_next.remote() for a in active]
    473                     # Always yield after each round of gets with timeout.

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\_private\client_mode_hook.py in wrapper(*args, **kwargs)
    103             if func.__name__ != "init" or is_client_mode_enabled_by_default:
    104                 return getattr(ray, func.__name__)(*args, **kwargs)
--> 105         return func(*args, **kwargs)
    106 
    107     return wrapper

~\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\worker.py in get(object_refs, timeout)
   1711                     worker.core_worker.dump_object_store_memory_usage()
   1712                 if isinstance(value, RayTaskError):
-> 1713                     raise value.as_instanceof_cause()
   1714                 else:
   1715                     raise value

RayTaskError(ValueError): ray::RolloutWorker.par_iter_next() (pid=14152, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x000001E6D6681C10>)
  File "python\ray\_raylet.pyx", line 625, in ray._raylet.execute_task
  File "python\ray\_raylet.pyx", line 629, in ray._raylet.execute_task
  File "python\ray\_raylet.pyx", line 578, in ray._raylet.execute_task.function_executor
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\_private\function_manager.py", line 609, in actor_method_executor
    return method(__ray_actor, *args, **kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\tracing\tracing_helper.py", line 451, in _resume_span
    return method(self, *_args, **_kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py", line 1151, in par_iter_next
    return next(self.local_it)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 381, in gen_rollouts
    yield self.sample()
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\tracing\tracing_helper.py", line 451, in _resume_span
    return method(self, *_args, **_kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 757, in sample
    batches = [self.input_reader.next()]
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 103, in next
    batches = [self.get_data()]
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 265, in get_data
    item = next(self._env_runner)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 656, in _env_runner
    eval_results = _do_policy_eval(
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 1070, in _do_policy_eval
    policy.compute_actions_from_input_dict(
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\policy\tf_policy.py", line 297, in compute_actions_from_input_dict
    fetched = builder.get(to_fetch)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\utils\tf_run_builder.py", line 48, in get
    raise e
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\utils\tf_run_builder.py", line 42, in get
    self._executed = run_timeline(
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\utils\tf_run_builder.py", line 92, in run_timeline
    fetches = sess.run(ops, feed_dict=feed_dict)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\tensorflow\python\client\session.py", line 970, in run
    result = self._run(None, fetches, feed_dict, options_ptr,
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\tensorflow\python\client\session.py", line 1167, in _run
    raise ValueError(
ValueError: Cannot feed value of shape (1, 1, 1) for Tensor policy_0_wk2/obs:0, which has shape (?, 1)
2022-02-20 22:58:58,980	ERROR worker.py:84 -- Unhandled error (suppress with RAY_IGNORE_UNHANDLED_ERRORS=1): ray::RolloutWorker.par_iter_next() (pid=3876, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x0000020FA0851C10>)
  File "python\ray\_raylet.pyx", line 625, in ray._raylet.execute_task
  File "python\ray\_raylet.pyx", line 629, in ray._raylet.execute_task
  File "python\ray\_raylet.pyx", line 578, in ray._raylet.execute_task.function_executor
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\_private\function_manager.py", line 609, in actor_method_executor
    return method(__ray_actor, *args, **kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\tracing\tracing_helper.py", line 451, in _resume_span
    return method(self, *_args, **_kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py", line 1151, in par_iter_next
    return next(self.local_it)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 381, in gen_rollouts
    yield self.sample()
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\tracing\tracing_helper.py", line 451, in _resume_span
    return method(self, *_args, **_kwargs)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 757, in sample
    batches = [self.input_reader.next()]
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 103, in next
    batches = [self.get_data()]
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 265, in get_data
    item = next(self._env_runner)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 656, in _env_runner
    eval_results = _do_policy_eval(
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 1070, in _do_policy_eval
    policy.compute_actions_from_input_dict(
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\policy\tf_policy.py", line 297, in compute_actions_from_input_dict
    fetched = builder.get(to_fetch)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\utils\tf_run_builder.py", line 48, in get
    raise e
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\utils\tf_run_builder.py", line 42, in get
    self._executed = run_timeline(
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\utils\tf_run_builder.py", line 92, in run_timeline
    fetches = sess.run(ops, feed_dict=feed_dict)
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\tensorflow\python\client\session.py", line 970, in run
    result = self._run(None, fetches, feed_dict, options_ptr,
  File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\tensorflow\python\client\session.py", line 1167, in _run
    raise ValueError(
ValueError: Cannot feed value of shape (1, 1, 1) for Tensor policy_0_wk1/obs:0, which has shape (?, 1)
8

@amohazab

That error is saying you are passing in an observation of shape [1,1,1] but it should be [1,1] so somewhere you are adding an extra dimension.

9

Do you want to create the preprocessor for debugging?

10

That is true. I though that maybe it is because of the counter variable, so I removed it. As the result, we will only have the episodes of length 1. It might have solved that issue but then, I received the this:

RayTaskError(ValueError): ray::RolloutWorker.par_iter_next() (pid=9296, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x000001504D2DEC10>)
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\tree\__init__.py", line 510, in map_structure
   [func(*args) for args in zip(*map(flatten, structures))])
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\tree\__init__.py", line 510, in <listcomp>
   [func(*args) for args in zip(*map(flatten, structures))])
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\policy\sample_batch.py", line 212, in concat_key
   return concat_aligned(values, time_major)
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\utils\numpy.py", line 97, in concat_aligned
   np.concatenate(items, out=output, axis=1 if time_major else 0)
 File "<__array_function__ internals>", line 5, in concatenate
TypeError: Cannot cast array data from dtype('O') to dtype('float32') according to the rule 'same_kind'

During handling of the above exception, another exception occurred:

ray::RolloutWorker.par_iter_next() (pid=9296, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x000001504D2DEC10>)
 File "python\ray\_raylet.pyx", line 618, in ray._raylet.execute_task
 File "python\ray\_raylet.pyx", line 659, in ray._raylet.execute_task
 File "python\ray\_raylet.pyx", line 625, in ray._raylet.execute_task
 File "python\ray\_raylet.pyx", line 629, in ray._raylet.execute_task
 File "python\ray\_raylet.pyx", line 578, in ray._raylet.execute_task.function_executor
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\_private\function_manager.py", line 609, in actor_method_executor
   return method(__ray_actor, *args, **kwargs)
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\tracing\tracing_helper.py", line 451, in _resume_span
   return method(self, *_args, **_kwargs)
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\iter.py", line 1151, in par_iter_next
   return next(self.local_it)
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 381, in gen_rollouts
   yield self.sample()
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\util\tracing\tracing_helper.py", line 451, in _resume_span
   return method(self, *_args, **_kwargs)
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 757, in sample
   batches = [self.input_reader.next()]
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 103, in next
   batches = [self.get_data()]
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 265, in get_data
   item = next(self._env_runner)
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 633, in _env_runner
   _process_observations(
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\sampler.py", line 1019, in _process_observations
   sample_collector.try_build_truncated_episode_multi_agent_batch()
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\collectors\simple_list_collector.py", line 886, in try_build_truncated_episode_multi_agent_batch
   batch = self._build_multi_agent_batch(episode=episode)
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\collectors\simple_list_collector.py", line 844, in _build_multi_agent_batch
   ma_batch[pid] = collector.build()
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\evaluation\collectors\simple_list_collector.py", line 441, in build
   batch = SampleBatch.concat_samples(self.batches)
 File "C:\Users\mohazaa1\AppData\Local\Programs\Python\Python39\lib\site-packages\ray\rllib\policy\sample_batch.py", line 223, in concat_samples
   raise ValueError(f"Cannot concat data under key '{k}', b/c "
ValueError: Cannot concat data under key 'rewards', b/c sub-structures under that key don't match.
11

@amohazab

Try printing your rewards dictionary in the environment step method. The error is indicating you have mixed types.

12

Yes. I wanted to call check_shape function.

13

@mannyv @Lars_Simon_Zehnder
It’s been resolved guys.
Thank you SO MUCH!

1 Like