Transport endpoint is not connected

[whoisalbertmatt]

I am a google colab pro+ user. I am trying to train a DL model and getting an error of "Transport endpoint is not connected", while training the model. Traceback (most recent call last): File "/usr/lib/python3.10/threading.py", line 1016, in _bootstrap_inner 19% 57100/300000 [3:52:57<16:30:59, 4.09it/s] Traceback (most recent call last): File "/content/drive/MyDrive/Ub4D/exp_runner.py", line 998, in <module> File "/content/drive/MyDrive/Ub4D/exp_runner.py", line 256, in train File "/usr/local/lib/python3.10/dist-packages/torch/utils/tensorboard/writer.py", line 391, in add_scalar self.run() File "/usr/local/lib/python3.10/dist-packages/tensorboard/summary/writer/event_file_writer.py", line 244, in run self._run() File "/usr/local/lib/python3.10/dist-packages/tensorboard/summary/writer/event_file_writer.py", line 275, in _run self._get_file_writer().add_summary(summary, global_step, walltime) self._record_writer.write(data) File "/usr/local/lib/python3.10/dist-packages/torch/utils/tensorboard/writer.py", line 113, in add_summary File "/usr/local/lib/python3.10/dist-packages/tensorboard/summary/writer/record_writer.py", line 40, in write self.add_event(event, global_step, walltime) File "/usr/local/lib/python3.10/dist-packages/torch/utils/tensorboard/writer.py", line 98, in add_event self._writer.write(header + header_crc + data + footer_crc) self.event_writer.add_event(event) File "/usr/local/lib/python3.10/dist-packages/tensorflow/python/lib/io/file_io.py", line 101, in write File "/usr/local/lib/python3.10/dist-packages/tensorboard/summary/writer/event_file_writer.py", line 117, in add_event self._writable_file.append( self._async_writer.write(event.SerializeToString()) tensorflow.python.framework.errors_impl.FailedPreconditionError: exp/Cactus/paper_config/logs/events.out.tfevents.1691087896.d58d3853234e.7295.0; Transport endpoint is not connected File "/usr/local/lib/python3.10/dist-packages/tensorboard/summary/writer/event_file_writer.py", line 171, in write self._check_worker_status() File "/usr/local/lib/python3.10/dist-packages/tensorboard/summary/writer/event_file_writer.py", line 212, in _check_worker_status raise exception File "/usr/lib/python3.10/threading.py", line 1016, in _bootstrap_inner self.run() File "/usr/local/lib/python3.10/dist-packages/tensorboard/summary/writer/event_file_writer.py", line 244, in run self._run() File "/usr/local/lib/python3.10/dist-packages/tensorboard/summary/writer/event_file_writer.py", line 275, in _run self._record_writer.write(data) File "/usr/local/lib/python3.10/dist-packages/tensorboard/summary/writer/record_writer.py", line 40, in write self._writer.write(header + header_crc + data + footer_crc) File "/usr/local/lib/python3.10/dist-packages/tensorflow/python/lib/io/file_io.py", line 101, in write self._writable_file.append( tensorflow.python.framework.errors_impl.FailedPreconditionError: exp/Cactus/paper_config/logs/events.out.tfevents.1691087896.d58d3853234e.7295.0; Transport endpoint is not connected