IndexError while training

[flammified]

trafficstars

Hi,

I am trying to use this repo to train a model with the following command:

However, I am getting the following error: python3 run_benchmark.py --sources simsat --no_relu --imerg --inc_time --gpus 1 --config_file config.yml --batch_size 8

Traceback (most recent call last):
  File "run_benchmark.py", line 313, in <module>
    main(hparams)
  File "run_benchmark.py", line 182, in main
    trainer.fit(model)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 771, in fit
    self._call_and_handle_interrupt(
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 724, in _call_and_handle_interrupt
    return trainer_fn(*args, **kwargs)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 812, in _fit_impl
    results = self._run(model, ckpt_path=self.ckpt_path)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1237, in _run
    results = self._run_stage()
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1324, in _run_stage
    return self._run_train()
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1354, in _run_train
    self.fit_loop.run()
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/loops/base.py", line 204, in run
    self.advance(*args, **kwargs)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/loops/fit_loop.py", line 269, in advance
    self._outputs = self.epoch_loop.run(self._data_fetcher)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/loops/base.py", line 204, in run
    self.advance(*args, **kwargs)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 171, in advance
    batch = next(data_fetcher)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 184, in __next__
    return self.fetching_function()
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 259, in fetching_function
    self._fetch_next_batch(self.dataloader_iter)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 273, in _fetch_next_batch
    batch = next(iterator)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/supporters.py", line 553, in __next__
    return self.request_next_batch(self.loader_iters)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/supporters.py", line 565, in request_next_batch
    return apply_to_collection(loader_iters, Iterator, next)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/apply_func.py", line 99, in apply_to_collection
    return function(data, *args, **kwargs)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 530, in __next__
    data = self._next_data()
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1224, in _next_data
    return self._process_data(data)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1250, in _process_data
    data.reraise()
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/torch/_utils.py", line 457, in reraise
    raise exception
IndexError: Caught IndexError in DataLoader worker process 7.
Original Traceback (most recent call last):
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py", line 287, in _worker_loop
    data = fetcher.fetch(index)
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 49, in fetch
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/home/alexander/PyRain/venv/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 49, in <listcomp>
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/home/alexander/PyRain/src/dataloader/memmap_dataloader.py", line 575, in __getitem__
    sample_results = self.get_sample_at(sample_mode_id, sample_ts, index)
  File "/home/alexander/PyRain/src/dataloader/memmap_dataloader.py", line 413, in get_sample_at
    self.dataset[(vbl_t + sample_ts,
  File "/home/alexander/PyRain/src/dataloader/memmap_dataloader.py", line 250, in __getitem__
    return results_dict[list(results_dict.keys())[0]]
IndexError: list index out of range

Can you perhaps give me a pointer on how to fix this?

Kind regards

Alexander Freeman