spikeinterface
spikeinterface copied to clipboard
Published
20 hours ago •
SpikeInterface
SpikeInterface
OSError: too many open files
Hi,
I tried to extract waveform from one sorted session, and it keeps crashing:
> 20231201_az_WDAN07 imec0.ap waveforms not extracted, extracting now. [210/738]
extract waveforms shared_memory multi buffer: 100%|██████████████████████████████| 3645/3645 [00:00<00:00, 8958.39it/s]
extract waveforms shared_memory multi buffer: 100%|█████████████████████████████| 3645/3645 [00:00<00:00, 10124.65it/s]
extract waveforms shared_memory multi buffer: 100%|█████████████████████████████| 3645/3645 [00:00<00:00, 10273.34it/s]
extract waveforms shared_memory multi buffer: 100%|██████████████████████████████| 3645/3645 [00:00<00:00, 8616.66it/s]
extract waveforms shared_memory multi buffer: 100%|█████████████████████████████| 3645/3645 [00:00<00:00, 10356.94it/s]
extract waveforms shared_memory multi buffer: 100%|█████████████████████████████| 3645/3645 [00:00<00:00, 10295.60it/s]
---------------------------------------------------------------------------
AssertionError Traceback (most recent call last)
File /opt/neuropixels/pixels/pixels/behaviours/base.py:916, in Behaviour.sort_spikes(self, CatGT_app, old)
915 try:
--> 916 waveforms = si.WaveformExtractor.load_from_folder(
917 folder=cache,
918 sorting=ks3_output,
919 )
920 print(f"> {self.name} {stream_id} waveforms extracted, now it is loaded.\n")
File ~/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py:141, in WaveformExtractor.load_from_folder(cls, folder, with_recording, sorting)
140 folder = Path(folder)
--> 141 assert folder.is_dir(), f"This waveform folder does not exists {folder}"
143 if not with_recording:
144 # load
AssertionError: This waveform folder does not exists /home/amz/interim/npx/20231201_az_WDAN07/cache_0
During handling of the above exception, another exception occurred:
OSError Traceback (most recent call last)
File ~/Documents/git/analysis/analysis/preprocessing/preprocessing.py:31, in <module>
17 """
18 # step 1: do CatGT
19 az_utils.run_catgt(
(...)
27
28 """
29 # step 2: spikesorting
30 #exp.sort_spikes()
---> 31 exp.sort_spikes(CatGT_app=catgt_dir)
32 stop = timeit.default_timer()
33 print(f"> Spike sorting of {exp} took {(stop - start)/60} mins.")
File /opt/neuropixels/pixels/pixels/experiment.py:126, in Experiment.sort_spikes(self, CatGT_app)
123 for i, session in enumerate(self.sessions)
124 print(">>>>> Sorting spikes for session {} ({} / {})"
125 .format(session.name, i + 1, len(self.sessions)))
--> 126 session.sort_spikes(CatGT_app=CatGT_app)
File /opt/neuropixels/pixels/pixels/behaviours/base.py:925, in Behaviour.sort_spikes(self, CatGT_app, old)
922 print(f"> {self.name} {stream_id} waveforms not extracted, extracting now.\n")
923 #if ks3_output.count_total_num_spikes()
924 # extract waveforms
--> 925 waveforms = si.extract_waveforms(
926 recording=concat_rec, #recording=test, # for testing
927 sorting=ks3_output,
928 folder=cache,
929 load_if_exists=True, # load extracted if available
930 #load_if_exists=False, # re-calculate everytime
931 max_spikes_per_unit=500, # None will extract all waveforms
932 ms_before=2.0, # time before trough
933 ms_after=3.0, # time after trough
934 overwrite=False,
935 #overwrite=True,
936 **job_kwargs,
937 )
939 """
940 # TODO: remove redundant units by keeping minimum shift, highest_amplitude, or
941 # max_spikes
(...)
946 )
947 """
948 # export to phy, with pc feature calculated.
949 # copy recording.dat to output so that individual waveforms can be
950 # seen in waveformview.
File ~/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py:1643, in extract_waveforms(recording, sorting, folder, mode, precompute_template, ms_before, ms_after, max_spikes_per_unit, overwrite, return
_scaled, dtype, sparse, sparsity, sparsity_temp_folder, num_spikes_for_sparsity, unit_batch_size, allow_unfiltered, use_relative_path, seed, load_if_exists, **kwargs)
1627 we = WaveformExtractor.create(
1628 recording,
1629 sorting,
(...)
1634 sparsity=sparsity,
1635 )
1636 we.set_params(
1637 ms_before=ms_before,
1638 ms_after=ms_after,
(...)
1641 return_scaled=return_scaled,
1642 )
-> 1643 we.run_extract_waveforms(seed=seed, **job_kwargs)
1645 if precompute_template is not None:
1646 we.precompute_templates(modes=precompute_template)
File ~/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py:1402, in WaveformExtractor.run_extract_waveforms(self, seed, **job_kwargs)
1399 else:
1400 sparsity_mask = self.sparsity.mask
-> 1402 wfs_arrays = extract_waveforms_to_buffers(
1403 self.recording,
1404 spikes,
1405 unit_ids,
1406 nbefore,
1407 nafter,
1408 mode=mode,
1409 return_scaled=return_scaled,
1410 folder=wf_folder,
1411 dtype=p["dtype"],
1412 sparsity_mask=sparsity_mask,
1413 copy=copy,
1414 **job_kwargs,
1415 )
1416 if self.folder is None:
1417 self._memory_objects["wfs_arrays"] = wfs_arrays
File ~/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_tools.py:92, in extract_waveforms_to_buffers(recording, spikes, unit_ids, nbefore, nafter, mode, return_scaled, folder, dtype, sparsity_mask, copy, **job_
kwargs)
89 dtype = "float32"
90 dtype = np.dtype(dtype)
---> 92 waveforms_by_units, arrays_info = allocate_waveforms_buffers(
93 recording, spikes, unit_ids, nbefore, nafter, mode=mode, folder=folder, dtype=dtype, sparsity_mask=sparsity_mask
94 )
96 distribute_waveforms_to_buffers(
97 recording,
98 spikes,
(...)
106 **job_kwargs,
107 )
109 if mode == "memmap":
File ~/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_tools.py:188, in allocate_waveforms_buffers(recording, spikes, unit_ids, nbefore, nafter, mode, folder, dtype, sparsity_mask)
186 if mode == "memmap":
187 filename = str(folder / f"waveforms_{unit_id}.npy")
--> 188 arr = np.lib.format.open_memmap(filename, mode="w+", dtype=dtype, shape=shape)
189 waveforms_by_units[unit_id] = arr
190 arrays_info[unit_id] = filename
File ~/.conda/envs/pixels/lib/python3.10/site-packages/numpy/lib/format.py:885, in open_memmap(filename, mode, dtype, shape, fortran_order, version)
882 if mode == 'w+':
883 mode = 'r+'
--> 885 marray = numpy.memmap(filename, dtype=dtype, shape=shape, order=order,
886 mode=mode, offset=offset)
888 return marray
File ~/.conda/envs/pixels/lib/python3.10/site-packages/numpy/core/memmap.py:267, in memmap.__new__(subtype, filename, dtype, mode, offset, shape, order)
265 bytes -= start
266 array_offset = offset - start
--> 267 mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start)
269 self = ndarray.__new__(subtype, shape, dtype=descr, buffer=mm,
270 offset=array_offset, order=order)
271 self._mmap = mm
OSError: [Errno 24] Too many open files
Traceback (most recent call last):
File "/opt/neuropixels/pixels/pixels/behaviours/base.py", line 916, in sort_spikes
waveforms = si.WaveformExtractor.load_from_folder(
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py", line 141, in load_from_folder
assert folder.is_dir(), f"This waveform folder does not exists {folder}"
AssertionError: This waveform folder does not exists /home/amz/interim/npx/20231201_az_WDAN07/cache_0
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/shellapp.py", line 435, in _run_cmd_line_code
self._exec_file(fname, shell_futures=True)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/shellapp.py", line 360, in _exec_file
self.shell.safe_execfile(full_filename,
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 2738, in safe_execfile
py3compat.execfile(
File "/home/amz/.local/lib/python3.10/site-packages/IPython/utils/py3compat.py", line 55, in execfile
exec(compiler(f.read(), fname, "exec"), glob, loc)
File "/home/amz/Documents/git/analysis/analysis/preprocessing/preprocessing.py", line 31, in <module>
exp.sort_spikes(CatGT_app=catgt_dir)
File "/opt/neuropixels/pixels/pixels/experiment.py", line 126, in sort_spikes
session.sort_spikes(CatGT_app=CatGT_app)
File "/opt/neuropixels/pixels/pixels/behaviours/base.py", line 925, in sort_spikes
waveforms = si.extract_waveforms(
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py", line 1643, in extract_waveforms
we.run_extract_waveforms(seed=seed, **job_kwargs)
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_extractor.py", line 1402, in run_extract_waveforms
wfs_arrays = extract_waveforms_to_buffers(
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_tools.py", line 92, in extract_waveforms_to_buffers
waveforms_by_units, arrays_info = allocate_waveforms_buffers(
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/spikeinterface/core/waveform_tools.py", line 188, in allocate_waveforms_buffers
arr = np.lib.format.open_memmap(filename, mode="w+", dtype=dtype, shape=shape)
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/numpy/lib/format.py", line 885, in open_memmap
marray = numpy.memmap(filename, dtype=dtype, shape=shape, order=order,
File "/home/amz/.conda/envs/pixels/lib/python3.10/site-packages/numpy/core/memmap.py", line 267, in __new__
mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start)
OSError: [Errno 24] Too many open files
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/amz/.conda/envs/pixels/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/amz/.local/lib/python3.10/site-packages/IPython/__init__.py", line 124, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/amz/.local/lib/python3.10/site-packages/traitlets/config/application.py", line 975, in launch_instance
app.initialize(argv)
File "/home/amz/.local/lib/python3.10/site-packages/traitlets/config/application.py", line 110, in inner
return method(app, *args, **kwargs)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/terminal/ipapp.py", line 285, in initialize
self.init_code()
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/shellapp.py", line 310, in init_code
self._run_cmd_line_code()
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/shellapp.py", line 437, in _run_cmd_line_code
self.shell.showtraceback(tb_offset=4)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 2007, in showtraceback
self.debugger(force=True)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 1107, in debugger
self.InteractiveTB.debugger(force=True)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 924, in debugger
self.pdb = self.debugger_cls()
File "/home/amz/.local/lib/python3.10/site-packages/IPython/terminal/debugger.py", line 29, in __init__
self.pt_init(pt_session_options)
File "/home/amz/.local/lib/python3.10/site-packages/IPython/terminal/debugger.py", line 90, in pt_init
self.pt_loop = asyncio.new_event_loop()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/events.py", line 782, in new_event_loop
return get_event_loop_policy().new_event_loop()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/events.py", line 673, in new_event_loop
return self._loop_factory()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/unix_events.py", line 64, in __init__
super().__init__(selector)
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/selector_events.py", line 56, in __init__
self._make_self_pipe()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/selector_events.py", line 103, in _make_self_pipe
self._ssock, self._csock = socket.socketpair()
File "/home/amz/.conda/envs/pixels/lib/python3.10/socket.py", line 607, in socketpair
a, b = _socket.socketpair(family, type, proto)
OSError: [Errno 24] Too many open files
If you suspect this is an IPython 8.4.0 bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]
You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.
Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True
Exception ignored in: <function BaseEventLoop.__del__ at 0x7f127c371750>
Traceback (most recent call last):
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/base_events.py", line 690, in __del__
self.close()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/unix_events.py", line 68, in close
super().close()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/selector_events.py", line 87, in close
self._close_self_pipe()
File "/home/amz/.conda/envs/pixels/lib/python3.10/asyncio/selector_events.py", line 94, in _close_self_pipe
self._remove_reader(self._ssock.fileno())
AttributeError: '_UnixSelectorEventLoop' object has no attribute '_ssock'
Could you please advise?
Thanks very much!
