Memory accumulation problem

[inchinn1]

I am struggling with a memory accumulation problem. I am running an alvium 1800 U-158m in hardware triggered mode. I am acquiring a series of frames periodically which is controlled by an external trigger. This all works fine.. except that for each loop the memory does seem to empty despite using gc.collect... eventually I run out of memory and can no longer acquire. Any help with this issue would be greatly appreciated. The code is pasted below.. I have tried inserting gc. collect in as many places as possible... but still get this problem.

import functools
import vimba
import time
import queue
import numpy as np
import threading
import tifffile as tf
import gc


def set_nearest_value(cam, feat_name: str, feat_value: int):
    # Helper function that tries to set a given value. If setting of the initial value failed
    # it calculates the nearest valid value and sets the result. This function is intended to
    # be used with Height and Width Features because not all Cameras allow the same values
    # for height and width.
    feat = cam.get_feature_by_name(feat_name)

    try:
        feat.set(feat_value)

    except vimba.VimbaFeatureError:
        min_, max_ = feat.get_range()
        inc = feat.get_increment()

        if feat_value <= min_:
            val = min_

        elif feat_value >= max_:
            val = max_

        else:
            val = (((feat_value - min_) // inc) * inc) + min_

        feat.set(val)

        msg = ('Camera {}: Failed to set value of Feature \'{}\' to \'{}\': '
               'Using nearest valid value \'{}\'. Note that, this causes resizing '
               'during processing, reducing the frame rate.')


def setup_mode(cam):
    with cam:
        # Set up recording mode
        try:
            set_nearest_value(cam, 'AcquisitionMode', 'Continuous')
            set_nearest_value(cam, 'TriggerSelector', 'FrameStart')
            set_nearest_value(cam, 'TriggerActivation', 'RisingEdge')
            set_nearest_value(cam, 'TriggerSource', 'Line0')
            set_nearest_value(cam, 'TriggerMode', 'On')
        except (AttributeError, vimba.VimbaFeatureError):
            pass


class ImageRecorder:
    def __init__(self, cam: vimba.Camera, frame_queue: queue.Queue):
        self._cam = cam
        self._queue = frame_queue
        self.killswitch = threading.Event()

    def __call__(self, cam: vimba.Camera, frame: vimba.Frame):
        # Place the image data as an opencv image and the frame ID into the queue
        self._queue.put((frame.as_opencv_image(), frame.get_timestamp()),)
        # Hand used frame back to Vimba so it can store the next image in this memory
        cam.queue_frame(frame)

    def stop(self,):
        self._cam.stop_streaming()  # check how this will work
        self.killswitch.set()
        gc.collect()

    def record_images(self, ):
        # This method assumes software trigger is desired. Free run image acquisition would work
        # similarly to get higher fps from the camera
        with vimba.Vimba.get_instance():
            with self._cam:
                try:
                    self._cam.start_streaming(handler=self)
                    self.killswitch.wait()
                    gc.collect()

                finally:
                    self._cam.stop_streaming()
                    gc.collect()


def write_image(rec, frame_queue: queue.Queue, data, ticks, nFrames):
    while True:
        # Get an element from the queue.
        for i in range(nFrames):
            frame, ts = frame_queue.get(block=True)
            ticks[i] = ts
            data[i, :, :] = frame[:, :, 0].astype('uint8')
            frame_queue.task_done()
            if i == nFrames-1:
                print('saving')
                rec.stop()
                gc.collect()


def main(nFrames, path, fname, data, ticks):
    # nFrames=280
    FRAME_HEIGHT = 1088
    FRAME_WIDTH = 1456

    file = path+fname

    with vimba.Vimba.get_instance() as vmb:
        cams = vmb.get_all_cameras()

        frame_queue = queue.Queue()
        recorder = ImageRecorder(cam=cams[0], frame_queue=frame_queue)
        # Start a thread that runs write_image(frame_queue). Marking it as daemon allows the python
        # program to exit even though that thread is still running. The thread will then be stopped
        # when the program exits
        threading.Thread(target=functools.partial(
            write_image, recorder, frame_queue, data, ticks, nFrames), daemon=True).start()

        recorder.record_images()

        frame_queue.join()

        start = time.time()
        tf.imwrite(file+'.tif', data)  # , compression= 'jpeg')
        print('saving took ' + str(time.time()-start)+' seconds')
        # ticks are 1GHz convert to seconds
        np.savetxt(file+'.txt', ticks/1000000000)
        del vmb, frame_queue, data, ticks, recorder
        gc.collect()
    return


def loop(name, n, nFrames):

    # nFrames = 1000
    path = 'D:/Data'
    # name = 'test'

    FRAME_HEIGHT = 1088
    FRAME_WIDTH = 1456

    with vimba.Vimba.get_instance() as vmb:
        cams = vmb.get_all_cameras()
        setup_mode(cams[0])

    for i in range(n):
        data = np.zeros((nFrames, FRAME_HEIGHT, FRAME_WIDTH), dtype='uint8')
        ticks = np.zeros(nFrames)
        print('getting ready for recording trial '+str(i))
        main(nFrames, path, name+str(i), data, ticks)
        del data, ticks,
        gc.collect()
        print('finished trial '+str(i))
    print('all trials finished')

[Teresa-AlliedVision]

Hello inchinn1, I had a quick look at the code with my debugger and it showed that the threads are not terminated after the image is saved and a new thread is started. I only let it run with 10 active threads or so, so I didn't get a crash yet. Try this best practice hardware-trigger example as a base without threads:

HardwareTriggerUSB.zip

Hope that helps, Teresa