FileCache.get_file error: No space left on device

[albertz]

...
  File "/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/tools/returnn/returnn/util/file_cache.py", line 334, in FileCache.handle_cached_files_in_config.<local
s>._handle_value
    line: res = self.get_file(value.filename)
    locals:
      res = <not found>
      self = <local> <returnn.util.file_cache.FileCache object at 0x1495fe502c90>
      self.get_file = <local> <bound method FileCache.get_file of <returnn.util.file_cache.FileCache object at 0x1495fe502c90>>
      value = <local> CachedFile(filename='/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/work/i6_core/returnn/oggzip/BlissToOggZipJob.5ad18raRAWhr/output/ou
t.ogg.zip')
      value.filename = <local> '/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/work/i6_core/returnn/oggzip/BlissToOggZipJob.5ad18raRAWhr/output/out.ogg.zip',
 len = 136
  File "/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/tools/returnn/returnn/util/file_cache.py", line 128, in FileCache.get_file
    line: raise last_error
    locals:
      last_error = <local> OSError(28, 'No space left on device')
  File "/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/tools/returnn/returnn/util/file_cache.py", line 120, in FileCache.get_file
    line: self._copy_file_if_needed(src_filename, dst_filename)
    locals:
      self = <not found>
      self._copy_file_if_needed = <not found>
      src_filename = <not found>
      dst_filename = <not found>
  File "/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/tools/returnn/returnn/util/file_cache.py", line 381, in FileCache._copy_file_if_needed
    line: with LockFile(
              directory=lock_dir, name=lock_file, lock_timeout=self._lock_timeout
          ) as lock, self._touch_files_thread.files_added_context(lock.lockfile):
    locals:
      LockFile = <global> <class 'returnn.util.basic.LockFile'>
      directory = <not found>
      lock_dir = <local> '/w0/tmp/slurm_az668407.57025316/az668407/returnn/file_cache/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/work/i6_core/returnn/oggzip/BlissToOggZipJob.5ad18raRAWhr/output', len = 183
      name = <not found>
      lock_file = <local> 'out.ogg.zip.returnn-lock', len = 24
      lock_timeout = <not found>
      self = <local> <returnn.util.file_cache.FileCache object at 0x1495fe502c90>
      self._lock_timeout = <local> 20
      lock = <not found>
      self._touch_files_thread = <local> <_TouchFilesThread(Thread-2, initial daemon)>
      self._touch_files_thread.files_added_context = <local> <bound method _TouchFilesThread.files_added_context of <_TouchFilesThread(Thread-2, initial daemon)>>
      lock.lockfile = <not found>
  File "/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/tools/returnn/returnn/util/basic.py", line 3212, in LockFile.__enter__
    line: self.lock()
    locals:
      self = <local> <returnn.util.basic.LockFile object at 0x1495fe4bc2f0>
      self.lock = <local> <bound method LockFile.lock of <returnn.util.basic.LockFile object at 0x1495fe4bc2f0>>
  File "/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/tools/returnn/returnn/util/basic.py", line 3159, in LockFile.lock
    line: if self.try_lock():
    locals:
      self = <local> <returnn.util.basic.LockFile object at 0x1495fe4bc2f0>
      self.try_lock = <local> <bound method LockFile.try_lock of <returnn.util.basic.LockFile object at 0x1495fe4bc2f0>>
  File "/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/tools/returnn/returnn/util/basic.py", line 3194, in LockFile.try_lock
    line: self.fd = os.open(self.lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR)
    locals:
      self = <local> <returnn.util.basic.LockFile object at 0x1495fe4bc2f0>
      self.fd = <local> None
      os = <global> <module 'os' (frozen)>
      os.open = <global> <built-in function open>
      self.lockfile = <local> '/w0/tmp/slurm_az668407.57025316/az668407/returnn/file_cache/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/work/i6_core/returnn
/oggzip/BlissToOggZipJob.5ad18raRAWhr/output/out.ogg.zip.returnn-lock', len = 208
      os.O_CREAT = <global> 64
      os.O_EXCL = <global> 128
      os.O_RDWR = <global> 2
OSError: [Errno 28] No space left on device: '/w0/tmp/slurm_az668407.57025316/az668407/returnn/file_cache/rwthfs/rz/cluster/home/az668407/setups/combined/2021-05-31/work/i6_core/returnn/oggzip/BlissToOggZipJob.5ad18raRAWhr/output/out.ogg.zip.returnn-lock'

So this error occurs while creating the lock file.

I'm not sure what's the most reasonable way to handle this. But anyway, I just want to report this for now.