Hello, I would like some help in order to find the root cause of
(bundle_reservation_check_func pid=37108) 2023-10-06 10:40:36,982 ERROR worker.py:782 -- Worker exits with an exit code 1.
(bundle_reservation_check_func pid=37108) Traceback (most recent call last):
(bundle_reservation_check_func pid=37108) File "python/ray/_raylet.pyx", line 1999, in ray._raylet.task_execution_handler
(bundle_reservation_check_func pid=37108) File "python/ray/_raylet.pyx", line 1894, in ray._raylet.execute_task_with_cancellation_handler
(bundle_reservation_check_func pid=37108) File "python/ray/_raylet.pyx", line 1558, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37108) File "python/ray/_raylet.pyx", line 1559, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37108) File "python/ray/_raylet.pyx", line 1562, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37108) File "python/ray/_raylet.pyx", line 1590, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37108) File "python/ray/_raylet.pyx", line 1592, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37108) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/worker.py", line 727, in deserialize_objects
(bundle_reservation_check_func pid=37108) context = self.get_serialization_context()
(bundle_reservation_check_func pid=37108) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(bundle_reservation_check_func pid=37108) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/worker.py", line 615, in get_serialization_context
(bundle_reservation_check_func pid=37108) context_map[job_id] = serialization.SerializationContext(self)
(bundle_reservation_check_func pid=37108) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(bundle_reservation_check_func pid=37108) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/serialization.py", line 153, in __init__
(bundle_reservation_check_func pid=37108) serialization_addons.apply(self)
(bundle_reservation_check_func pid=37108) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/util/serialization_addons.py", line 86, in apply
(bundle_reservation_check_func pid=37108) _register_custom_datasets_serializers(serialization_context)
(bundle_reservation_check_func pid=37108) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/arrow_serialization.py", line 46, in _register_custom_datasets_serializers
(bundle_reservation_check_func pid=37108) import pyarrow as pa # noqa: F401
(bundle_reservation_check_func pid=37108) ^^^^^^^^^^^^^^^^^^^^
(bundle_reservation_check_func pid=37108) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/pyarrow/__init__.py", line 65, in <module>
(bundle_reservation_check_func pid=37108) import pyarrow.lib as _lib
(bundle_reservation_check_func pid=37108) File "pyarrow/lib.pyx", line 24, in init pyarrow.lib
(bundle_reservation_check_func pid=37108) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/numpy/__init__.py", line 144, in <module>
(bundle_reservation_check_func pid=37108) from . import lib
(bundle_reservation_check_func pid=37108) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/numpy/lib/__init__.py", line 39, in <module>
(bundle_reservation_check_func pid=37108) from . import _version
(bundle_reservation_check_func pid=37108) File "<frozen importlib._bootstrap>", line 1176, in _find_and_load
(bundle_reservation_check_func pid=37108) File "<frozen importlib._bootstrap>", line 1137, in _find_and_load_unlocked
(bundle_reservation_check_func pid=37108) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/worker.py", line 779, in sigterm_handler
(bundle_reservation_check_func pid=37108) sys.exit(1)
(bundle_reservation_check_func pid=37108) SystemExit: 1
(bundle_reservation_check_func pid=37108) [2023-10-06 10:40:37,011 C 37108 37108] core_worker.cc:791: Check failed: _s.ok() Bad status: IOError: Broken pipe
(bundle_reservation_check_func pid=37108) *** StackTrace Information ***
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0xf11d1a) [0x7f37bbf11d1a] ray::operator<<()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0xf13802) [0x7f37bbf13802] ray::SpdLogMessage::Flush()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x37) [0x7f37bbf13b17] ray::RayLog::~RayLog()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker4ExitENS_3rpc14WorkerExitTypeERKSsRKSt10shared_ptrINS_17LocalMemoryBufferEE+0x2bb) [0x7f37bb73ea5b] ray::core::CoreWorker::Exit()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker11ExecuteTaskERKNS_17TaskSpecificationERKSt10shared_ptrISt13unordered_mapISsSt6vectorISt4pairIldESaIS9_EESt4hashISsESt8equal_toISsESaIS8_IKSsSB_EEEEPS7_IS8_INS_8ObjectIDES5_INS_9RayObjectEEESaISQ_EEST_PS7_IS8_ISN_bESaISU_EEPN6google8protobuf16RepeatedPtrFieldINS_3rpc20ObjectReferenceCountEEEPbPSs+0x1b7e) [0x7f37bb74f2be] ray::core::CoreWorker::ExecuteTask()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZNSt17_Function_handlerIFN3ray6StatusERKNS0_17TaskSpecificationESt10shared_ptrISt13unordered_mapISsSt6vectorISt4pairIldESaIS9_EESt4hashISsESt8equal_toISsESaIS8_IKSsSB_EEEEPS7_IS8_INS0_8ObjectIDES5_INS0_9RayObjectEEESaISO_EESR_PS7_IS8_ISL_bESaISS_EEPN6google8protobuf16RepeatedPtrFieldINS0_3rpc20ObjectReferenceCountEEEPbPSsESt5_BindIFMNS0_4core10CoreWorkerEFS1_S4_RKSK_SR_SR_SV_S12_S13_S14_EPS18_St12_PlaceholderILi1EES1E_ILi2EES1E_ILi3EES1E_ILi4EES1E_ILi5EES1E_ILi6EES1E_ILi7EES1E_ILi8EEEEE9_M_invokeERKSt9_Any_dataS4_OSK_OSR_S1U_OSV_OS12_OS13_OS14_+0x58) [0x7f37bb682cd8] std::_Function_handler<>::_M_invoke()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x776b84) [0x7f37bb776b84] ray::core::CoreWorkerDirectTaskReceiver::HandleTask()::{lambda()#1}::operator()()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x777e8a) [0x7f37bb777e8a] std::_Function_handler<>::_M_invoke()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x78f54e) [0x7f37bb78f54e] ray::core::InboundRequest::Accept()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x75e600) [0x7f37bb75e600] ray::core::NormalSchedulingQueue::ScheduleRequests()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0xa3cd86) [0x7f37bba3cd86] EventTracker::RecordExecution()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0xa2ea5e) [0x7f37bba2ea5e] std::_Function_handler<>::_M_invoke()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0xa2efb6) [0x7f37bba2efb6] boost::asio::detail::completion_handler<>::do_complete()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x10132bb) [0x7f37bc0132bb] boost::asio::detail::scheduler::do_run_one()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x1015299) [0x7f37bc015299] boost::asio::detail::scheduler::run()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x10159a2) [0x7f37bc0159a2] boost::asio::io_context::run()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker20RunTaskExecutionLoopEv+0x1c) [0x7f37bb717f8c] ray::core::CoreWorker::RunTaskExecutionLoop()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core21CoreWorkerProcessImpl26RunWorkerTaskExecutionLoopEv+0x8c) [0x7f37bb75975c] ray::core::CoreWorkerProcessImpl::RunWorkerTaskExecutionLoop()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core17CoreWorkerProcess20RunTaskExecutionLoopEv+0x1d) [0x7f37bb75990d] ray::core::CoreWorkerProcess::RunTaskExecutionLoop()
(bundle_reservation_check_func pid=37108) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x5779d7) [0x7f37bb5779d7] __pyx_pw_3ray_7_raylet_10CoreWorker_7run_task_loop()
(bundle_reservation_check_func pid=37108) ray::IDLE() [0x5352a0] method_vectorcall_NOARGS
(bundle_reservation_check_func pid=37108) ray::IDLE(PyObject_Vectorcall+0x31) [0x51bde1] PyObject_Vectorcall
(bundle_reservation_check_func pid=37108) ray::IDLE(_PyEval_EvalFrameDefault+0x753) [0x50ed83] _PyEval_EvalFrameDefault
(bundle_reservation_check_func pid=37108) ray::IDLE() [0x5c83fe] _PyEval_Vector
(bundle_reservation_check_func pid=37108) ray::IDLE(PyEval_EvalCode+0x9f) [0x5c7aff] PyEval_EvalCode
(bundle_reservation_check_func pid=37108) ray::IDLE() [0x5e87b7] run_eval_code_obj
(bundle_reservation_check_func pid=37108) ray::IDLE() [0x5e4df0] run_mod
(bundle_reservation_check_func pid=37108) ray::IDLE() [0x5f90e2] pyrun_file
(bundle_reservation_check_func pid=37108) ray::IDLE(_PyRun_SimpleFileObject+0x19f) [0x5f86cf] _PyRun_SimpleFileObject
(bundle_reservation_check_func pid=37108) ray::IDLE(_PyRun_AnyFileObject+0x43) [0x5f8423] _PyRun_AnyFileObject
(bundle_reservation_check_func pid=37108) ray::IDLE(Py_RunMain+0x2ee) [0x5f2f9e] Py_RunMain
(bundle_reservation_check_func pid=37108) ray::IDLE(Py_BytesMain+0x39) [0x5b6f49] Py_BytesMain
(bundle_reservation_check_func pid=37108) /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7f37bd029d90]
(bundle_reservation_check_func pid=37108) /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x80) [0x7f37bd029e40] __libc_start_main
(bundle_reservation_check_func pid=37108) ray::IDLE() [0x5b6d9f]
(bundle_reservation_check_func pid=37108)
(bundle_reservation_check_func pid=37191) 2023-10-06 10:40:37,332 ERROR worker.py:782 -- Worker exits with an exit code 1.
(bundle_reservation_check_func pid=37191) Traceback (most recent call last):
(bundle_reservation_check_func pid=37191) File "python/ray/_raylet.pyx", line 1999, in ray._raylet.task_execution_handler
(bundle_reservation_check_func pid=37191) File "python/ray/_raylet.pyx", line 1894, in ray._raylet.execute_task_with_cancellation_handler
(bundle_reservation_check_func pid=37191) File "python/ray/_raylet.pyx", line 1558, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37191) File "python/ray/_raylet.pyx", line 1559, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37191) File "python/ray/_raylet.pyx", line 1562, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37191) File "python/ray/_raylet.pyx", line 1590, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37191) File "python/ray/_raylet.pyx", line 1592, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37191) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/worker.py", line 727, in deserialize_objects
(bundle_reservation_check_func pid=37191) context = self.get_serialization_context()
(bundle_reservation_check_func pid=37191) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(bundle_reservation_check_func pid=37191) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/worker.py", line 615, in get_serialization_context
(bundle_reservation_check_func pid=37191) context_map[job_id] = serialization.SerializationContext(self)
(bundle_reservation_check_func pid=37191) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(bundle_reservation_check_func pid=37191) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/serialization.py", line 153, in __init__
(bundle_reservation_check_func pid=37191) serialization_addons.apply(self)
(bundle_reservation_check_func pid=37191) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/util/serialization_addons.py", line 86, in apply
(bundle_reservation_check_func pid=37191) _register_custom_datasets_serializers(serialization_context)
(bundle_reservation_check_func pid=37191) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/arrow_serialization.py", line 46, in _register_custom_datasets_serializers
(bundle_reservation_check_func pid=37191) import pyarrow as pa # noqa: F401
(bundle_reservation_check_func pid=37191) ^^^^^^^^^^^^^^^^^^^^
(bundle_reservation_check_func pid=37191) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/pyarrow/__init__.py", line 65, in <module>
(bundle_reservation_check_func pid=37191) import pyarrow.lib as _lib
(bundle_reservation_check_func pid=37191) File "pyarrow/io.pxi", line 29, in init pyarrow.lib
(bundle_reservation_check_func pid=37191) File "<frozen io>", line 60, in __getattr__
(bundle_reservation_check_func pid=37191) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/worker.py", line 779, in sigterm_handler
(bundle_reservation_check_func pid=37191) sys.exit(1)
(bundle_reservation_check_func pid=37191) SystemExit: 1
(bundle_reservation_check_func pid=37191) [2023-10-06 10:40:37,365 C 37191 37191] core_worker.cc:791: Check failed: _s.ok() Bad status: IOError: Broken pipe
(bundle_reservation_check_func pid=37191) *** StackTrace Information ***
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0xf11d1a) [0x7f5401311d1a] ray::operator<<()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0xf13802) [0x7f5401313802] ray::SpdLogMessage::Flush()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x37) [0x7f5401313b17] ray::RayLog::~RayLog()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker4ExitENS_3rpc14WorkerExitTypeERKSsRKSt10shared_ptrINS_17LocalMemoryBufferEE+0x2bb) [0x7f5400b3ea5b] ray::core::CoreWorker::Exit()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker11ExecuteTaskERKNS_17TaskSpecificationERKSt10shared_ptrISt13unordered_mapISsSt6vectorISt4pairIldESaIS9_EESt4hashISsESt8equal_toISsESaIS8_IKSsSB_EEEEPS7_IS8_INS_8ObjectIDES5_INS_9RayObjectEEESaISQ_EEST_PS7_IS8_ISN_bESaISU_EEPN6google8protobuf16RepeatedPtrFieldINS_3rpc20ObjectReferenceCountEEEPbPSs+0x1b7e) [0x7f5400b4f2be] ray::core::CoreWorker::ExecuteTask()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZNSt17_Function_handlerIFN3ray6StatusERKNS0_17TaskSpecificationESt10shared_ptrISt13unordered_mapISsSt6vectorISt4pairIldESaIS9_EESt4hashISsESt8equal_toISsESaIS8_IKSsSB_EEEEPS7_IS8_INS0_8ObjectIDES5_INS0_9RayObjectEEESaISO_EESR_PS7_IS8_ISL_bESaISS_EEPN6google8protobuf16RepeatedPtrFieldINS0_3rpc20ObjectReferenceCountEEEPbPSsESt5_BindIFMNS0_4core10CoreWorkerEFS1_S4_RKSK_SR_SR_SV_S12_S13_S14_EPS18_St12_PlaceholderILi1EES1E_ILi2EES1E_ILi3EES1E_ILi4EES1E_ILi5EES1E_ILi6EES1E_ILi7EES1E_ILi8EEEEE9_M_invokeERKSt9_Any_dataS4_OSK_OSR_S1U_OSV_OS12_OS13_OS14_+0x58) [0x7f5400a82cd8] std::_Function_handler<>::_M_invoke()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x776b84) [0x7f5400b76b84] ray::core::CoreWorkerDirectTaskReceiver::HandleTask()::{lambda()#1}::operator()()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x777e8a) [0x7f5400b77e8a] std::_Function_handler<>::_M_invoke()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x78f54e) [0x7f5400b8f54e] ray::core::InboundRequest::Accept()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x75e600) [0x7f5400b5e600] ray::core::NormalSchedulingQueue::ScheduleRequests()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0xa3cd86) [0x7f5400e3cd86] EventTracker::RecordExecution()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0xa2ea5e) [0x7f5400e2ea5e] std::_Function_handler<>::_M_invoke()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0xa2efb6) [0x7f5400e2efb6] boost::asio::detail::completion_handler<>::do_complete()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x10132bb) [0x7f54014132bb] boost::asio::detail::scheduler::do_run_one()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x1015299) [0x7f5401415299] boost::asio::detail::scheduler::run()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x10159a2) [0x7f54014159a2] boost::asio::io_context::run()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker20RunTaskExecutionLoopEv+0x1c) [0x7f5400b17f8c] ray::core::CoreWorker::RunTaskExecutionLoop()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core21CoreWorkerProcessImpl26RunWorkerTaskExecutionLoopEv+0x8c) [0x7f5400b5975c] ray::core::CoreWorkerProcessImpl::RunWorkerTaskExecutionLoop()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core17CoreWorkerProcess20RunTaskExecutionLoopEv+0x1d) [0x7f5400b5990d] ray::core::CoreWorkerProcess::RunTaskExecutionLoop()
(bundle_reservation_check_func pid=37191) /home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_raylet.so(+0x5779d7) [0x7f54009779d7] __pyx_pw_3ray_7_raylet_10CoreWorker_7run_task_loop()
(bundle_reservation_check_func pid=37191) ray::IDLE() [0x5352a0] method_vectorcall_NOARGS
(bundle_reservation_check_func pid=37191) ray::IDLE(PyObject_Vectorcall+0x31) [0x51bde1] PyObject_Vectorcall
(bundle_reservation_check_func pid=37191) ray::IDLE(_PyEval_EvalFrameDefault+0x753) [0x50ed83] _PyEval_EvalFrameDefault
(bundle_reservation_check_func pid=37191) ray::IDLE() [0x5c83fe] _PyEval_Vector
(bundle_reservation_check_func pid=37191) ray::IDLE(PyEval_EvalCode+0x9f) [0x5c7aff] PyEval_EvalCode
(bundle_reservation_check_func pid=37191) ray::IDLE() [0x5e87b7] run_eval_code_obj
(bundle_reservation_check_func pid=37191) ray::IDLE() [0x5e4df0] run_mod
(bundle_reservation_check_func pid=37191) ray::IDLE() [0x5f90e2] pyrun_file
(bundle_reservation_check_func pid=37191) ray::IDLE(_PyRun_SimpleFileObject+0x19f) [0x5f86cf] _PyRun_SimpleFileObject
(bundle_reservation_check_func pid=37191) ray::IDLE(_PyRun_AnyFileObject+0x43) [0x5f8423] _PyRun_AnyFileObject
(bundle_reservation_check_func pid=37191) ray::IDLE(Py_RunMain+0x2ee) [0x5f2f9e] Py_RunMain
(bundle_reservation_check_func pid=37191) ray::IDLE(Py_BytesMain+0x39) [0x5b6f49] Py_BytesMain
(bundle_reservation_check_func pid=37191) /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7f5402629d90]
(bundle_reservation_check_func pid=37191) /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x80) [0x7f5402629e40] __libc_start_main
(bundle_reservation_check_func pid=37191) ray::IDLE() [0x5b6d9f]
(bundle_reservation_check_func pid=37191)
2023-10-06 10:40:37,373 ERROR worker.py:405 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): The worker died unexpectedly while executing this task. Check python-core-worker-*.log files for more information.
2023-10-06 10:40:39,384 ERROR worker.py:405 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): The worker died unexpectedly while executing this task. Check python-core-worker-*.log files for more information.
(bundle_reservation_check_func pid=37775) 2023-10-06 10:40:39,380 ERROR worker.py:782 -- Worker exits with an exit code 1.
(bundle_reservation_check_func pid=37775) Traceback (most recent call last):
(bundle_reservation_check_func pid=37775) File "python/ray/_raylet.pyx", line 1999, in ray._raylet.task_execution_handler
(bundle_reservation_check_func pid=37775) File "python/ray/_raylet.pyx", line 1894, in ray._raylet.execute_task_with_cancellation_handler
(bundle_reservation_check_func pid=37775) File "python/ray/_raylet.pyx", line 1558, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37775) File "python/ray/_raylet.pyx", line 1559, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37775) File "python/ray/_raylet.pyx", line 1562, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37775) File "python/ray/_raylet.pyx", line 1590, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37775) File "python/ray/_raylet.pyx", line 1592, in ray._raylet.execute_task
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/worker.py", line 727, in deserialize_objects
(bundle_reservation_check_func pid=37775) context = self.get_serialization_context()
(bundle_reservation_check_func pid=37775) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/worker.py", line 615, in get_serialization_context
(bundle_reservation_check_func pid=37775) context_map[job_id] = serialization.SerializationContext(self)
(bundle_reservation_check_func pid=37775) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/serialization.py", line 153, in __init__
(bundle_reservation_check_func pid=37775) serialization_addons.apply(self)
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/util/serialization_addons.py", line 86, in apply
(bundle_reservation_check_func pid=37775) _register_custom_datasets_serializers(serialization_context)
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/arrow_serialization.py", line 46, in _register_custom_datasets_serializers
(bundle_reservation_check_func pid=37775) import pyarrow as pa # noqa: F401
(bundle_reservation_check_func pid=37775) ^^^^^^^^^^^^^^^^^^^^
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/pyarrow/__init__.py", line 65, in <module>
(bundle_reservation_check_func pid=37775) import pyarrow.lib as _lib
(bundle_reservation_check_func pid=37775) File "pyarrow/lib.pyx", line 24, in init pyarrow.lib
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/numpy/__init__.py", line 144, in <module>
(bundle_reservation_check_func pid=37775) from . import lib
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/numpy/lib/__init__.py", line 25, in <module>
(bundle_reservation_check_func pid=37775) from . import index_tricks
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/numpy/lib/index_tricks.py", line 12, in <module>
(bundle_reservation_check_func pid=37775) import numpy.matrixlib as matrixlib
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/numpy/matrixlib/__init__.py", line 4, in <module>
(bundle_reservation_check_func pid=37775) from . import defmatrix
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/numpy/matrixlib/defmatrix.py", line 12, in <module>
(bundle_reservation_check_func pid=37775) from numpy.linalg import matrix_power
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/numpy/linalg/__init__.py", line 73, in <module>
(bundle_reservation_check_func pid=37775) from . import linalg
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/numpy/linalg/linalg.py", line 37, in <module>
(bundle_reservation_check_func pid=37775) from numpy._typing import NDArray
(bundle_reservation_check_func pid=37775) File "<frozen importlib._bootstrap>", line 1176, in _find_and_load
(bundle_reservation_check_func pid=37775) File "<frozen importlib._bootstrap>", line 1138, in _find_and_load_unlocked
(bundle_reservation_check_func pid=37775) File "<frozen importlib._bootstrap>", line 1078, in _find_spec
(bundle_reservation_check_func pid=37775) File "<frozen importlib._bootstrap_external>", line 1504, in find_spec
(bundle_reservation_check_func pid=37775) File "<frozen importlib._bootstrap_external>", line 1476, in _get_spec
(bundle_reservation_check_func pid=37775) File "<frozen importlib._bootstrap_external>", line 1612, in find_spec
(bundle_reservation_check_func pid=37775) File "<frozen importlib._bootstrap_external>", line 147, in _path_stat
(bundle_reservation_check_func pid=37775) File "/home/dlovison/miniconda3/envs/cerbero/lib/python3.11/site-packages/ray/_private/worker.py", line 779, in sigterm_handler
(bundle_reservation_check_func pid=37775) sys.exit(1)
(bundle_reservation_check_func pid=37775) SystemExit: 1
2023-10-06 10:40:42,998 WARNING tune.py:194 -- Stop signal received (e.g. via SIGINT/Ctrl+C), ending Ray Tune run. This will try to checkpoint the experiment state one last time. Press CTRL+C (or send SIGINT/SIGKILL/SIGTERM) to skip.
Process finished with exit code 137 (interrupted by signal 9: SIGKILL)
``