I try to run ray memory command from CLI, but I get the following error.
ray memory
Traceback (most recent call last):
File "/scratch/pyvenv/bin/ray", line 8, in <module>
sys.exit(main())
File "/scratch/pyvenv/lib64/python3.8/site-packages/ray/scripts/scripts.py", line 2422, in main
return cli()
File "/scratch/pyvenv/lib64/python3.8/site-packages/click/core.py", line 1137, in __call__
return self.main(*args, **kwargs)
File "/scratch/pyvenv/lib64/python3.8/site-packages/click/core.py", line 1062, in main
rv = self.invoke(ctx)
File "/scratch/pyvenv/lib64/python3.8/site-packages/click/core.py", line 1668, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/scratch/pyvenv/lib64/python3.8/site-packages/click/core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/scratch/pyvenv/lib64/python3.8/site-packages/click/core.py", line 763, in invoke
return __callback(*args, **kwargs)
File "/scratch/pyvenv/lib64/python3.8/site-packages/ray/scripts/scripts.py", line 1872, in memory
mem_stats = memory_summary(
File "/scratch/pyvenv/lib64/python3.8/site-packages/ray/_private/internal_api.py", line 40, in memory_summary
return memory_summary(
File "/scratch/pyvenv/lib64/python3.8/site-packages/ray/dashboard/memory_utils.py", line 399, in memory_summary
node_stats(raylet["NodeManagerAddress"], raylet["NodeManagerPort"])
File "/scratch/pyvenv/lib64/python3.8/site-packages/ray/_private/internal_api.py", line 101, in node_stats
node_stats = stub.GetNodeStats(
File "/scratch/pyvenv/lib64/python3.8/site-packages/grpc/_channel.py", line 1161, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/scratch/pyvenv/lib64/python3.8/site-packages/grpc/_channel.py", line 1004, in _end_unary_response_blocking
raise _InactiveRpcError(state) # pytype: disable=not-instantiable
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "failed to connect to all addresses; last error: UNKNOWN: ipv4:10.254.203.53:80: connection attempt timed out before receiving SETTINGS frame"
debug_error_string = "UNKNOWN:failed to connect to all addresses; last error: UNKNOWN: ipv4:10.254.203.53:80: connection attempt timed out before receiving SETTINGS frame {grpc_status:14, created_time:"2023-09-07T21:44:56.935922178+00:00"}"
Please let me know how can I solve this?