Hey there,
I was wondering whether I should do something more than shown in the tutorial if you are working with an IterableDataset. I am already adhering to these guidelines from Pytorch Lightning, but I am receiving the following stacktrace:
(pid=10422) GPU available: False, used: False
(pid=10422) TPU available: None, using: 0 TPU cores
(pid=10422)
(pid=10422) | Name | Type | Params
(pid=10422) ----------------------------------------------------
(pid=10422) 0 | conv1 | Conv2d | 21.0 K
(pid=10422) 1 | conv2 | Conv2d | 5.3 M
(pid=10422) 2 | conv3 | Conv2d | 10.6 M
(pid=10422) 3 | flatten | Flatten | 0
(pid=10422) 4 | fc1 | Linear | 6.6 M
(pid=10422) 5 | fc2 | Linear | 513
(pid=10422) 6 | loss | BCEWithLogitsLoss | 0
(pid=10422) 7 | train_metrics | ModuleDict | 0
(pid=10422) 8 | val_metrics | ModuleDict | 0
(pid=10422) 9 | test_metrics | ModuleDict | 0
(pid=10422) ----------------------------------------------------
(pid=10422) 22.5 M Trainable params
(pid=10422) 0 Non-trainable params
(pid=10422) 22.5 M Total params
(pid=10422) 2021-01-17 21:21:22,407 ERROR function_runner.py:254 -- Runner Thread raised error.
(pid=10422) Traceback (most recent call last):
(pid=10422) File "/user/yorick/.local/lib/python3.7/site-packages/ray/tune/function_runner.py", line 248, in run
(pid=10422) self._entrypoint()
(pid=10422) File "/user/yorick/.local/lib/python3.7/site-packages/ray/tune/function_runner.py", line 316, in entrypoint
(pid=10422) self._status_reporter.get_checkpoint())
(pid=10422) File "/user/yorick/.local/lib/python3.7/site-packages/ray/tune/function_runner.py", line 575, in _trainable_func
(pid=10422) output = fn()
(pid=10422) File "/user/yorick/.local/lib/python3.7/site-packages/ray/tune/function_runner.py", line 650, in _inner
(pid=10422) inner(config, checkpoint_dir=None)
(pid=10422) File "/user/yorick/.local/lib/python3.7/site-packages/ray/tune/function_runner.py", line 644, in inner
(pid=10422) fn(config, **fn_kwargs)
(pid=10422) File "<ipython-input-29-c17fd0385946>", line 41, in train_model
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 472, in fit
(pid=10422) results = self.accelerator_backend.train()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/accelerators/cpu_accelerator.py", line 62, in train
(pid=10422) results = self.train_or_test()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/accelerators/accelerator.py", line 69, in train_or_test
(pid=10422) results = self.trainer.train()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 523, in train
(pid=10422) self.train_loop.run_training_epoch()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/trainer/training_loop.py", line 565, in run_training_epoch
(pid=10422) for batch_idx, (batch, is_last_batch) in train_dataloader:
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/profiler/profilers.py", line 83, in profile_iterable
(pid=10422) value = next(iterator)
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 45, in _with_is_last
(pid=10422) it = iter(iterable)
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 352, in __iter__
(pid=10422) return self._get_iterator()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 294, in _get_iterator
(pid=10422) return _MultiProcessingDataLoaderIter(self)
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 827, in __init__
(pid=10422) self._reset(loader, first_iter=True)
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 857, in _reset
(pid=10422) self._try_put_index()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 1091, in _try_put_index
(pid=10422) index = self._next_index()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 427, in _next_index
(pid=10422) return next(self._sampler_iter) # may raise StopIteration
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/sampler.py", line 227, in __iter__
(pid=10422) for idx in self.sampler:
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/sampler.py", line 67, in __iter__
(pid=10422) return iter(range(len(self.data_source)))
(pid=10422) TypeError: object of type 'MarginsIterableDataset' has no len()
(pid=10422) Exception in thread Thread-2:
(pid=10422) Traceback (most recent call last):
(pid=10422) File "/usr/local/lib/python3.7/threading.py", line 926, in _bootstrap_inner
(pid=10422) self.run()
(pid=10422) File "/user/yorick/.local/lib/python3.7/site-packages/ray/tune/function_runner.py", line 267, in run
(pid=10422) raise e
(pid=10422) File "/user/yorick/.local/lib/python3.7/site-packages/ray/tune/function_runner.py", line 248, in run
(pid=10422) self._entrypoint()
(pid=10422) File "/user/yorick/.local/lib/python3.7/site-packages/ray/tune/function_runner.py", line 316, in entrypoint
(pid=10422) self._status_reporter.get_checkpoint())
(pid=10422) File "/user/yorick/.local/lib/python3.7/site-packages/ray/tune/function_runner.py", line 575, in _trainable_func
(pid=10422) output = fn()
(pid=10422) File "/user/yorick/.local/lib/python3.7/site-packages/ray/tune/function_runner.py", line 650, in _inner
(pid=10422) inner(config, checkpoint_dir=None)
(pid=10422) File "/user/yorick/.local/lib/python3.7/site-packages/ray/tune/function_runner.py", line 644, in inner
(pid=10422) fn(config, **fn_kwargs)
(pid=10422) File "<ipython-input-29-c17fd0385946>", line 41, in train_model
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 472, in fit
(pid=10422) results = self.accelerator_backend.train()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/accelerators/cpu_accelerator.py", line 62, in train
(pid=10422) results = self.train_or_test()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/accelerators/accelerator.py", line 69, in train_or_test
(pid=10422) results = self.trainer.train()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 523, in train
(pid=10422) self.train_loop.run_training_epoch()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/trainer/training_loop.py", line 565, in run_training_epoch
(pid=10422) for batch_idx, (batch, is_last_batch) in train_dataloader:
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/profiler/profilers.py", line 83, in profile_iterable
(pid=10422) value = next(iterator)
(pid=10422) File "/usr/local/lib/python3.7/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 45, in _with_is_last
(pid=10422) it = iter(iterable)
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 352, in __iter__
(pid=10422) return self._get_iterator()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 294, in _get_iterator
(pid=10422) return _MultiProcessingDataLoaderIter(self)
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 827, in __init__
(pid=10422) self._reset(loader, first_iter=True)
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 857, in _reset
(pid=10422) self._try_put_index()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 1091, in _try_put_index
(pid=10422) index = self._next_index()
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 427, in _next_index
(pid=10422) return next(self._sampler_iter) # may raise StopIteration
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/sampler.py", line 227, in __iter__
(pid=10422) for idx in self.sampler:
(pid=10422) File "/usr/local/lib/python3.7/site-packages/torch/utils/data/sampler.py", line 67, in __iter__
(pid=10422) return iter(range(len(self.data_source)))
(pid=10422) TypeError: object of type 'MarginsIterableDataset' has no len()
(pid=10422)
It is a CNN classification (2 classes) model. Running it without Raytune works fine. Currently running this on a CPU setup.
pytorch == 1.7.1
pytorch-lightning == 1.1.3
ray == 1.1.0