No Reward Appearing for MARL Environment during Training

Hi, so I successfully got RLLIB to run on an underlying environment I specified for it. However, the reward and episode mean max etc. appear only as nan. I thought this might be a numerical issue, but this doesn’t seem to be the issue and I am getting valid Q values. Do I need to specify the reward somewhere to the trainer? It is specified in the environment of course.

Additionally at the end I get

The full code is

import numpy as np
import gym
from gym import spaces
import numpy as np
from gym.utils import seeding, EzPickle
from ray.rllib.utils.typing import MultiAgentDict, AgentID
from typing import Tuple, Dict, List
from gym.envs.registration import EnvSpec
from ray.rllib.env.multi_agent_env import MultiAgentEnv
import ray
from ray.tune.logger import pretty_print
from ray.tune.registry import register_env
from ray.rllib.agents import ppo
from ray.rllib.agents import ddpg
import ray.tune as tune

INITIAL_ASSET_HOLDINGS = 1
BORROW_LIM = -0.01
R_VALUE = 1.03
DELTA = 0.01
W_VALUE = 0.98
GAMMA = 2
AGENT_NUM = 1
N = 5
BETA = 0.95
ALPHA = 0.33
Z = 1
np.random.seed(2020)

# alternative for raylib


class AiyagariEnvironment(gym.Env):
    """ An environment for value function sampling from a basic RA GE model with capital"""

    # resets state to initial value
    #  def u(cons):
    #      util = cons**(1-GAMMA)/(1-GAMMA)
    #      return util
    # idea pass assets to multiagent, and then return interest rate back to environment.
    metadata = {"render.modes": ["human"]}

    def __init__(self):
        super(AiyagariEnvironment, self).__init__()
        self.reward_range = (0, np.inf)
        self.seed()
        # next period asset space bounds [borrow_lim, inf)
        self.action_space = spaces.Box(
            low=np.array([BORROW_LIM]), high=np.array([np.inf]), dtype=np.float32
        )
        # observation space -- all variables agent will observe before making new decision. Since we assume r will be fixed here, this will include here assets, prices, income. Due to assets acting as summary statistic in this model we will only provide current period assets, prices, income. We can extend this to multi-period if we wanted.
        self.observation_space = spaces.Box(
            low=np.array([BORROW_LIM, 0, 0]),
            high=np.array([np.inf, np.inf, np.inf]),
            dtype=np.float32,
        )
        self.assets = INITIAL_ASSET_HOLDINGS
        self.price = R_VALUE - DELTA
        self.W = W_VALUE
        self.current_step = 0
        self.cons = 0
        self.net_worth = self.assets * self.price
        self.reward = 0
        self.shock = np.exp(self.np_random.normal(0, 1))
        self.income = self.W * self.shock
        self.obs = np.array([self.assets, self.price, self.income])
        self.current_step = 0

    # resets state to initial value
    #  def u(cons):
    #      util = cons**(1-GAMMA)/(1-GAMMA)
    #      return utily
    def seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]

    def reset(self):
        self.assets = INITIAL_ASSET_HOLDINGS
        self.price = R_VALUE - DELTA
        self.W = W_VALUE
        self.current_step = 0
        self.cons = 0
        self.net_worth = self.assets * self.price
        self.reward = 0
        self.shock = np.exp(self.np_random.normal(0, 1))
        self.income = self.W * self.shock
        self.obs = np.array([self.assets, self.price, self.income])
        self.current_step = 0
        # shifted exponential for time being, can impose own distribution with custom sampling later on.
        # for time being will use default distribution for sampling.
        return self.obs

    # updating function
    @property
    def n(self):
        return AGENT_NUM

    def step(self, action, R, W):
        self.current_step += 1
        self.price = R - DELTA
        self.W = W
        self.shock = np.exp(np.random.normal(0, 1))
        self.income = self.W * self.shock
        self.net_worth = (self.price) * self.assets + self.income
        if action in self.action_space:
            if action <= self.net_worth:
                self.assets = action + (self.price) * self.assets
                self.cons = self.net_worth - action
            else:
                self.assets = self.net_worth + (self.price) * self.assets
        else:
            raise ValueError(
                "Received invalid action={:f} which is not part of the action space".format(
                    action
                )
            )
        self.obs = np.array([self.assets, self.price, self.income])
        done = self.cons <= 0
        self.done = done
        if self.cons > 0:
            self.reward = (self.cons ** (1 - GAMMA) / (1 - GAMMA)).item()
        else:
            self.reward = -np.inf
        return self.obs, self.reward, self.done, {}

    def render(self, mode="human", close=False):
        # work on render to make graph.
        results = str(
            f"Step: {self.current_step}\n"
            f"Assets: {self.assets}\n"
            f"Income: {self.income}\n"
            f"Consumption: {self.cons}\n"
            f"Net worth: {self.net_worth}\n"
            f"Interest Rate: {self.price}\n"
            f"Wage Rate: {self.W}\n"
            f"Utility: {self.reward}\n"
        )
        return results


class AiyagariMultiAgentEnv(MultiAgentEnv):
    def __init__(self, num):
        self.agents = [AiyagariEnvironment() for _ in range(num)]
        self.dones = set()
        self.observation_space = gym.spaces.Box(
            low=np.array([BORROW_LIM, 0, 0, 0, 0, 0, 0]),
            high=np.array([np.inf, np.inf, np.inf, np.inf, np.inf, np.inf, np.inf]),
            dtype=np.float32,
        )
        self.action_space = gym.spaces.Box(
            low=np.array([BORROW_LIM]), high=np.array([np.inf]), dtype=np.float32
        )
        self.resetted = False
        self.num = num

    def reset(self):
        self.resetted = True
        self.dones = set()
        dict_agents = {i: np.zeros(7) for i, a in enumerate(self.agents)}
        # initial holdings
        self.K = sum(self.agents[i].assets for i in range(self.num))
        self.N = self.num
        self.R = Z * (1 - ALPHA) * (self.N / self.K) ** (ALPHA)
        self.W = Z * (ALPHA) * (self.K / self.N) ** (1 - ALPHA)
        agg_obs_list = [self.K, self.N, self.R, self.W]
        for i in range(self.num):
            dict_agents[i][0:3] = self.agents[i].reset()
            dict_agents[i][3:7] = np.array(agg_obs_list)
        return dict_agents

    def step(self, action_dict):
        obs, rew, done, info = {}, {}, {}, {}
        obs_temp_list = {}
        obs =  dict.fromkeys(range(5))
        obs_temp_list = dict.fromkeys(range(5))
        obs_temp_list = {i: self.agents[i].step(action,self.R,self.W)[0] for i, action in action_dict.items()}
        obs_temp = np.zeros(7)
        for i, action in action_dict.items():
            # get observations which is tomorrow's capital earnings. Use to construct tomorrow prices. then feedback in.
            obs_temp[0:3], rew[i], done[i], info[i] = self.agents[i].step(
                action, self.R, self.W
            ) 
            obs_temp_list[i] = obs_temp
            # append aggregate observations to each i.
            if done[i]:
                self.dones.add(i)
        # construct and append aggregate states
        self.K = sum(obs_val[0] for obs_val in obs_temp_list.values())
        self.N = self.num
        self.R = Z * (1 - ALPHA) * (self.N / self.K) ** (ALPHA)
        self.W = Z * (ALPHA) * (self.K / self.N) ** (1 - ALPHA)
        for i in range(0, 5):
            #print(i)
            #print(np.size(obs_temp_list[i]))
            obs_temp_list[i][3:7] = np.array([self.K, self.N, self.R, self.W])
            obs[i] = obs_temp_list[i]
        done["__all__"] = len(self.dones) == len(self.agents)
        return obs, rew, done, info

    def render(self, mode="human", close=True):
        # TODO: work on nice render
        results_n = []
        for agent in self.agents:
            # results += env.render(mode, close)
            results = agent.render(mode, close)
            results_n.append(results)
        return results_n


if __name__ == "__main__":

    env = AiyagariMultiAgentEnv(5)
    obs = env.reset()
    for items in env.render():
        print(f"Agent: {env.render().index(items)+1} \n")
        print(items)
    print(env.action_space)

    tune.register_env("my_env", lambda config: AiyagariMultiAgentEnv(5))

    #obs_space = env.observation_space
    #act_spc = env.action_space
    #policies = {agent: (None, obs_space, act_spc, {}) for agent in env.agents}
    ray.init()
    config = {
            "env": "my_env",
            # General
            "num_gpus": 0,
            "num_workers": 2,
            # Method specific
            }  
    analysis=tune.run(
         "DDPG",
        stop={"training_iteration": 100},
         checkpoint_freq=10,
        config=config,
        checkpoint_at_end = True
    )   
    checkpoints = analysis.get_trial_checkpoints_paths(trial= analysis.get_best_trial("epsiode_reward_mean"), metric="episode_reward_mean")
    agent = ddpg.DDPGTrainer(config=config, env="my_env")
    agent.restore(checkpoints)  
    # analysis=tune.run(
    #     "PPO",
    #     stop={"episodes_total": 60000},
    #     checkpoint_freq=10,
    #     config=config,
    #     checkpoint_at_end = True
    # )
    #checkpoints = analysis.get_trial_checkpoints_paths(trial= analysis.get_best_trial("epsiode_reward_mean"), metric="episode_reward_mean")

    #agent = ppo.PPOTrainer(config=config, env="my_env")
    #agent.restore(checkpoints)  
    episode_reward = 0
    done = False
    obs = env.reset()
    while not done:
        action = agent.compute_action(obs)
        obs, reward, done, info = env.step(action)
        episode_reward += reward

@sven1977 could you take a look?

I experience the same issue. The reward is available in the trainer but not in ray.tune

Could the problem be that you are returning inf at some steps?
Like this:

if self.cons > 0:
     self.reward = (self.cons ** (1 - GAMMA) / (1 - GAMMA)).item()
else:
     self.reward = -np.inf

I’m not sure, but the reward should be a number.

1 Like

Thanks for the input. I now have a wrapper checking the bounds for the reward.

PS If someone else experiences this, I also forgot to set “horizon” in the config. → episode_len_mean is nan as the environment might not finish (e.g. long running/ indefinite simulation).

Thanks for the catch @rsv!
Yes, rewards must always be non-inf (and non-nan :wink: ) floats.
Also, if episodes never end, tune will always report episode_reward_mean=NaN b/c it reports an average over all already finished(!) episodes.