Ddpg restore problem

I am new to the RLlib and is still learning how to use it.
When I try to restore my pretrained DDPG model from the check point, it raise an error as below
TypeError: <tf.Tensor ‘default_policy/ou_state:0’ shape=() dtype=resource> is out of scope and cannot be used here. Use return values, explicit Python locals or TensorFlow collections to access it.
Please see أداء أفضل مع وظيفة tf  |  TensorFlow Core for more information.

<tf.Tensor ‘default_policy/ou_state:0’ shape=() dtype=resource> was defined here:
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/runpy.py”, line 193, in _run_module_as_main
main”, mod_spec)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/runpy.py”, line 85, in _run_code
exec(code, run_globals)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ipykernel_launcher.py”, line 16, in
app.launch_new_instance()
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/traitlets/config/application.py”, line 846, in launch_instance
app.start()
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ipykernel/kernelapp.py”, line 677, in start
self.io_loop.start()
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tornado/platform/asyncio.py”, line 199, in start
self.asyncio_loop.run_forever()
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/asyncio/base_events.py”, line 541, in run_forever
self._run_once()
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/asyncio/base_events.py”, line 1786, in _run_once
handle._run()
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/asyncio/events.py”, line 88, in _run
self._context.run(self._callback, *self._args)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ipykernel/kernelbase.py”, line 471, in dispatch_queue
await self.process_one()
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ipykernel/kernelbase.py”, line 460, in process_one
await dispatch(*args)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ipykernel/kernelbase.py”, line 367, in dispatch_shell
await result
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ipykernel/kernelbase.py”, line 662, in execute_request
reply_content = await reply_content
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ipykernel/ipkernel.py”, line 360, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ipykernel/zmqshell.py”, line 532, in run_cell
return super().run_cell(*args, **kwargs)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/IPython/core/interactiveshell.py”, line 2915, in run_cell
raw_cell, store_history, silent, shell_futures)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/IPython/core/interactiveshell.py”, line 2960, in _run_cell
return runner(coro)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/IPython/core/async_helpers.py”, line 78, in pseudo_sync_runner
coro.send(None)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/IPython/core/interactiveshell.py”, line 3186, in run_cell_async
interactivity=interactivity, compiler=compiler, result=result)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/IPython/core/interactiveshell.py”, line 3377, in run_ast_nodes
if (await self.run_code(code, result, async
=asy)):
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/IPython/core/interactiveshell.py”, line 3457, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File “/tmp/ipykernel_3631/4187836080.py”, line 12, in
trainer=ddpg.DDPGTrainer(env=TrainEnv, config=model_config)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/agents/trainer.py”, line 747, in init
sync_function_tpl)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/tune/trainable.py”, line 124, in init
self.setup(copy.deepcopy(self.config))
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/agents/trainer.py”, line 827, in setup
num_workers=self.config[“num_workers”])
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/agents/trainer.py”, line 2002, in _make_workers
logdir=self.logdir,
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/evaluation/worker_set.py”, line 132, in init
spaces=spaces,
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/evaluation/worker_set.py”, line 540, in _make_worker
spaces=spaces,
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/evaluation/rollout_worker.py”, line 590, in init
seed=seed)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/evaluation/rollout_worker.py”, line 1578, in _build_policy_map
conf, merged_conf)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/policy/policy_map.py”, line 134, in create_policy
observation_space, action_space, merged_config)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/policy/tf_policy_template.py”, line 252, in init
get_batch_divisibility_req=get_batch_divisibility_req,
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/policy/dynamic_tf_policy.py”, line 271, in init
self.exploration = self._create_exploration()
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/policy/policy.py”, line 755, in create_exploration
self.config.get(“framework”, “tf”)))
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/utils/from_config.py”, line 192, in from_config
object
= constructor(*ctor_args, **ctor_kwargs)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/utils/exploration/ornstein_uhlenbeck_noise.py”, line 70, in init
device=None)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/ray/rllib/utils/framework.py”, line 240, in get_variable
“shape”: shape
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py”, line 1633, in get_variable
aggregation=aggregation)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py”, line 1349, in get_variable
aggregation=aggregation)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py”, line 604, in get_variable
aggregation=aggregation)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py”, line 556, in _true_getter
aggregation=aggregation)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py”, line 974, in _get_single_variable
aggregation=aggregation)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/util/traceback_utils.py”, line 150, in error_handler
return fn(*args, **kwargs)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/variables.py”, line 265, in call
return cls._variable_v1_call(*args, **kwargs)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/variables.py”, line 225, in _variable_v1_call
shape=shape)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/variables.py”, line 203, in
previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py”, line 2721, in default_variable_creator
shape=shape)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/util/traceback_utils.py”, line 150, in error_handler
return fn(*args, **kwargs)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/variables.py”, line 269, in call
return super(VariableMetaclass, cls).call(*args, **kwargs)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py”, line 1676, in init
distribute_strategy=distribute_strategy)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py”, line 1832, in _init_from_args
graph_mode=self._in_graph_mode)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py”, line 238, in eager_safe_variable_handle
graph_mode, initial_value)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py”, line 166, in _variable_handle_from_shape_and_dtype
container=container)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/ops/gen_resource_variable_ops.py”, line 1215, in var_handle_op
allowed_devices=allowed_devices, name=name)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py”, line 742, in _apply_op_helper
attrs=attr_protos, op_def=op_def)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/framework/ops.py”, line 3784, in _create_op_internal
op_def=op_def)
File “/home/qinmolei/anaconda3/envs/FinRL/lib/python3.7/site-packages/tensorflow/python/framework/ops.py”, line 2175, in init
self._traceback = tf_stack.extract_stack_for_node(self._c_op)

The tensor <tf.Tensor ‘default_policy/ou_state:0’ shape=() dtype=resource> cannot be accessed from here, because it was defined in <tensorflow.python.framework.ops.Graph object at 0x7fa6e00cc810>, which is out of scope.

Can any one tell me where did I go wrong? Thanks