Add custom policy to config on a non multi-agent setup

RLlib
1

I’m wondering how to alter the default policy in the PPOConfig. I’m aware it can be set up like so for a multiagent environment:

def policy_map_fn(agent_id: str, _episode=None, _worker=None, **_kwargs) -> str:
    """
    Maps agent_id to policy_id
    """
    return 'policy'

algo = (
    ppo.PPOConfig()
    .environment(MyEnv)
    .multi_agent(
        policies={
            "policy": (
                CustomPolicy,
                observation_space,
                ActionSpace(),
                ppo.PPOConfig.overrides(gamma=0.9),
            ),
        },
        policy_mapping_fn=policy_map_fn,
    )
    .framework("torch")
    .build()
)

I can’t see anywhere in the API or documentation though how to use your own custom policy in a non multiagent setup?

2
3

Thanks, I used this code to build my algorithm class:

class AlteredPPOAlgo(ppo.PPO):
    @classmethod
    def get_default_policy_class(cls, config):
        return CustomPolicy

config = (
    ppo.PPOConfig(AlteredPPOAlgo)
    .environment(MyEnv)
    .framework("torch")
)

config.model.update({
    "custom_model": "my_torch_model",
    "custom_action_dist": "my_dist",
})

algo = config.build()