billiard OSError: [Errno 24] Too many open files

Log:

File "./MRdata.py", line 395, in mp_listener tfile_bf.start(write_thn, tfn, write_bf, pfn=pageFN) File "./MRdata.py", line 267, in start self.thread[thn].start() File "/usr/local/python-3.5/lib/python3.5/site-packages/billiard/process.py", line 122, in start self._popen = self._Popen(self) File "/usr/local/python-3.5/lib/python3.5/site-packages/billiard/context.py", line 276, in _Popen return _default_context.get_context().Process._Popen(process_obj) File "/usr/local/python-3.5/lib/python3.5/site-packages/billiard/context.py", line 333, in _Popen return Popen(process_obj) File "/usr/local/python-3.5/lib/python3.5/site-packages/billiard/popen_fork.py", line 24, in init self._launch(process_obj) File "/usr/local/python-3.5/lib/python3.5/site-packages/billiard/popen_fork.py", line 71, in _launch parent_r, child_w = os.pipe() OSError: [Errno 24] Too many open files

Code:

from billiard.context import Process
......

class CSVWrite:

    def __init__(self):
        self.thread = {
            'a': Process(),
            'b': Process(),
            'c': Process(),
            'd': Process(),
            'e': Process()
        }
        self.th_default = list(self.thread)
        self.th_is_set = {i:True for i in self.th_default}
        self.wfobj = {}

    def FileOpen(self, wfn):
        self.wfobj[wfn] = open(......)

    def run(self, wfn, datas, pfn=None):
        file_name = pfn if pfn else wfn
        mgr_mp_process[file_name] = 1
        writer = DictWriter(self.wfobj[file_name], fieldnames=mgr_tfileh_dict[wfn])
        if mgr_tfile_dict[file_name] != 2:
            writer.writeheader()
            mgr_tfile_dict[file_name] = 2
        writer.writerows(datas)
        self.wfobj[file_name].flush()
        if file_name in mgr_mp_process: del mgr_mp_process[file_name]

    def start(self, thn, wfn, datas, pfn=None):
        if self.th_is_set[thn] is False:
            self.thread[thn] = Process()
        self.thread[thn]._target = self.run
        self.thread[thn].daemon = True
        self.thread[thn]._args = (wfn, datas,)
        self.thread[thn]._kwargs = {'pfn': pfn}
        self.thread[thn].start()
        self.th_is_set[thn] = False

    def is_alive(self, auto_plus=False, limit=10):
        if len(self.thread) > len(self.th_default):
            for thn in list(self.thread):
                if thn not in self.th_default and self.thread[thn].is_alive() is False:
                    self.thread[thn].close()
                    self.thread[thn].join()
                    del self.thread[thn]
                    del self.th_is_set[thn]
        if len(self.thread) >= limit:
            return None
        for thn in list(self.thread):
            if self.thread[thn].is_alive():
                continue
            else:
                return thn
        if auto_plus:
            auto_thn = RandomData.make_char(5)
            if auto_thn in self.thread: auto_thn = RandomData.make_char(8)
            self.thread[auto_thn] = None
            self.th_is_set[auto_thn] = False
            return auto_thn
        return None

    def th_kill(self):
        ......

    def FileClose(self):
        ......

Apr 14 '17 05:04 pashifika

does this still exists after upgrading to latest master?

Jul 17 '17 12:07 auvipy

is this resolved? we are still seeing this issue.

Jan 03 '19 00:01 dorbala

is this resolved? we are still seeing this issue.

Apr 10 '19 02:04 hewei1990102

I faced a similar issue. The problems is that billiard.Pool creates extra pipes in linux and then doesn't close them. My issue is here https://github.com/celery/billiard/issues/282