ContourArea causes program to crash

[iversionpeng]

question

• How does javaCv manage memory?
• How do we use these objects correctly?

maven

    <dependency>
        <groupId>org.bytedeco</groupId>
        <artifactId>javacv</artifactId>
        <version>1.5.9</version>
    </dependency>

    <dependency>
        <groupId>org.bytedeco</groupId>
        <artifactId>javacpp</artifactId>
        <classifier>macosx-x86_64</classifier>
        <version>1.5.9</version>
    </dependency>
    <dependency>
        <groupId>org.bytedeco</groupId>
        <artifactId>opencv</artifactId>
        <classifier>macosx-x86_64</classifier>
        <version>1.5.9</version>
    </dependency>

code

package com.debug2016.ocr.ofline;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bytedeco.javacpp.indexer.Indexer;
import org.bytedeco.opencv.opencv_core.*;
import org.opencv.core.Core;
import org.opencv.core.CvType;

import java.util.Arrays;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import static org.bytedeco.opencv.global.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_imgcodecs.imread;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
import static org.bytedeco.opencv.opencv_core.Mat.ones;


public class OriginLightCheck {
private static Logger log = LogManager.getLogger();

public static void main(String[] strings) {
    String imagePath = strings[0];
    Integer count = Integer.valueOf(strings[1]);

    long maxMemory = Runtime.getRuntime().maxMemory();
    long totalMemory = Runtime.getRuntime().totalMemory();
    long freeMemory = Runtime.getRuntime().freeMemory();
    System.out.println("Max Memory: " + maxMemory / 1024 / 1024 + "MB");
    System.out.println("Allocated Memory: " + totalMemory / 1024 / 1024 + "MB");
    System.out.println("Free Memory: " + freeMemory / 1024 / 1024 + "MB");
    log.info("testSdk begin ossKey={}", imagePath);
    Mat test = imread(imagePath);
    if (test == null || test.cols() <= 1) {
        log.warn("testSdk mat downloadMat error");
        return;
    }
    for (int j = 0; j < count; j++) {
        try {
            Mat clone = test.clone();
            ImageCheckResult testSdk = securityCheck(clone, j + "");
            log.info("testSdk-" + Thread.currentThread().getName() + "-" + j + "-" + testSdk);
        } catch (Exception e) {
            log.warn("testSdk mat processing error", e);
        }
    }
}

public static void securityCheck(Mat originMat, String sessionId) {
    log.info("securityCheck {} a originMat={},hash={},reference={}", sessionId, originMat, originMat.hashCode(), originMat.referenceCount());

    Mat hsvMat = new Mat();
    cvtColor(originMat, hsvMat, COLOR_BGR2HSV);

    Scalar lowerBound1 = new Scalar(0, 80, 250, 0);
    Scalar upperBound1 = new Scalar(180, 255, 255, 0); 
    Mat securityMat = new Mat();
    Mat low1 = new Mat(lowerBound1);
    Mat upper1 = new Mat(upperBound1);
    inRange(hsvMat, low1, upper1, securityMat);

    MatExpr securityKernel = ones(5, 5, CV_8UC1);
    Mat securityDilateMat = new Mat();
    dilate(securityMat, securityDilateMat, securityKernel.asMat(), new Point(-1, -1), 3, BORDER_CONSTANT, morphologyDefaultBorderValue());

    MatExpr kernel = ones(3, 3, CV_8UC1);
    Mat securityErodeMat = new Mat();
    erode(securityDilateMat, securityErodeMat, kernel.asMat(), new Point(-1, -1), 4, BORDER_CONSTANT, morphologyDefaultBorderValue());

    MatVector contours = new MatVector();
    Mat hierarchy = new Mat();
    findContours(
            securityErodeMat,
            contours,
            hierarchy,
            RETR_CCOMP,
            CHAIN_APPROX_SIMPLE
    );
    log.info("securityCheck {} contours this={},hash={},reference={}", sessionId, contours, contours.address());
   
    Mat[] mats = contours.get();
    AtomicInteger num = new AtomicInteger(0);
    List<Mat> collect = Arrays.stream(mats).peek(a -> {
        num.getAndIncrement();
        log.info("securityCheck {} newMat count={},Mat={},hash={},reference={},contoursHash={},", sessionId, num.get(), a, a.address(), a.referenceCount());
    }).sorted((o1, o2) -> {
        double area1 = contourArea(o1, false);
        double area2 = contourArea(o2, false);
        return Double.compare(area2, area1);
    }).collect(Collectors.toList());
    
    boolean haveLight = false;
    int loopCount = Math.min(collect.size(), 4);
    for (int i = 0; i < loopCount; ++i) {
        Mat mat = collect.get(i);
        log.info("securityCheck {} preCheck count={},Mat={},hash={},reference={},contoursHash={}", sessionId, i, mat, mat.address(), mat.referenceCount());
    }
    for (int i = 0; i < loopCount; ++i) {
        Mat myMat = collect.get(i);
        try {
            log.info("securityCheck {} start count={},myMat={},hash={},reference={},contoursHash={}", sessionId, i, myMat, myMat.address(), myMat.referenceCount());

            Mat mask = new Mat(originMat.size(), CV_8UC1, Scalar.all(0));
            log.info("securityCheck {} mask count={},myMat={},hash={},reference={},contoursHash={}", sessionId, i, myMat, myMat.address(), myMat.referenceCount());

            drawContours(mask, new MatVector(myMat.clone()), 0, new Scalar(255, 255, 255, 0));
            log.info("securityCheck {} drawContours count={},myMat={},hash={},reference={},contoursHash={}", sessionId, i, myMat, myMat.address(), myMat.referenceCount());

            Mat extractedRegion = new Mat();
            log.info("securityCheck {} extractedRegion count={},myMat={},hash={},reference={},contoursHash={}", sessionId, i, myMat, myMat.address(), myMat.referenceCount());

            hsvMat.copyTo(extractedRegion, mask);
            log.info("securityCheck {} copyTo count={},myMat={},hash={},reference={},contoursHash={}", sessionId, i, myMat, myMat.address(), myMat.referenceCount());

            Map<Scalar, Integer> scalarIntegerMap = checkLightNum(extractedRegion, myMat);
            log.info("securityCheck {} checkLightNum count={},myMat={},hash={},reference={},contoursHash={}", sessionId, i, myMat, myMat.address(), myMat.referenceCount());

            double area = contourArea(myMat, false);
        } catch (Exception e) {
            log.info("securityCheck {} error1 count={},myMat={},hash={},reference={},contoursHash={}", sessionId, i, myMat, myMat.address(), myMat.referenceCount(),e);
        }
        log.info("securityCheck {} end count={},myMat={},hash={},reference={},contoursHash={}", sessionId, i, myMat, myMat.address(), myMat.referenceCount());
    }
    log.info("securityCheck {} end", sessionId);
}

private static Map<Scalar, Integer> checkLightNum(Mat extractedRegion, Mat myMat) {

    log.info("checkLightNum 01 myMat={},hash={},myMatreference={},contoursHash={}", myMat, myMat.address(), myMat.referenceCount());

    List<Scalar> detectedColors = new ArrayList<>();
    Map<Scalar, Integer> colorMap = new HashMap<>();
    Indexer indexer = extractedRegion.createIndexer();
    log.info("checkLightNum 02 myMat={},hash={},myMatreference={},contoursHash={}", myMat, myMat.address(), myMat.referenceCount());


    Boolean isLog = false;
    for (int y = 0; y < extractedRegion.rows(); y++) {
        for (int x = 0; x < extractedRegion.cols(); x++) {
          
            Scalar currentColor = new Scalar(h, s, v, 0);
          if ((myMat.cols()) == 0 && !isLog) {
                isLog = true;
                log.info("checkLightNum findChange myMat={},hash={},myMatreference={},contoursHash={}", myMat, myMat.address(), myMat.referenceCount());
            }
        }
    }
//        detectedColors.forEach(Pointer::deallocate);
//        colorMap.keySet().forEach(Pointer::close);
    log.info("checkLightNum end myMat={},hash={},myMatreference={},contoursHash={}", myMat, myMat.address(), myMat.referenceCount());
    return colorMap;
     }

}

log

1. securityCheck 256 contours this=org.bytedeco.opencv.opencv_core.Mat[width=1,height=209,depth=-2147483616,channels=2]],address= 0x600002457dc0,reference={}
2. checkLightNum 02 myMat=org.bytedeco.opencv.opencv_core.Mat[width=1,height=209,depth=-2147483616,channels=2],address=140261930922848,myMatreference=-1,contoursHash={}
3. [GC (Allocation Failure) 267734K->14246K(595968K), 0.0013573 secs]
4. Debug: Collecting org.bytedeco.javacpp.Pointer$NativeDeallocator[ownerAddress=0x600002457dc0,deallocatorAddress=0x12647c310]
5. checkLightNum end myMat=org.bytedeco.opencv.opencv_core.Mat[width=10,height=19576,depth=-2147483640,channels=17],address=140261930922848,myMatreference=-1,contoursHash={}
6. securityCheck 256 checkLightNum count=1,myMat=org.bytedeco.opencv.opencv_core.Mat[width=10,height=19576,depth=-2147483640,channels=17],address=140261930922848,reference=-1,contoursHash={}

Log conclusion: MatVector(address = 0x600002457dc0), Mat(address=140261930922848). From the above log, it can be seen that before MatVector is deallocated, Mat[width=1,height=209,depth=-2147483616,channels=2], After MatVector deallocate Mat[width=10,height=19576,depth=-2147483640,channels=17]

I am using a Mat object, but changes have occurred inside it, causing my program to crash? Why is it designed like this? How can I avoid this problem?

hs_error_pid.log

`SIGSEGV (0xb) at pc=0x0000000124db5edf, pid=5705, tid=0x0000000000001203 Stack: [0x000000030d836000,0x000000030d936000], sp=0x000000030d9354e0, free space=1021k Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)

• C [libopencv_core.407.dylib+0x12cedf] cv::Mat::Mat(cv::Mat const&)+0x4f
• C [libopencv_imgproc.407.dylib+0x278c93] cv::contourArea(cv::_InputArray const&, bool)+0x43
• C [libjniopencv_imgproc.dylib+0x93571]
• Java_org_bytedeco_opencv_global_opencv_1imgproc_contourArea__Lorg_bytedeco_opencv_opencv_1core_Mat_2Z+0x81
• J 1370 org.bytedeco.opencv.global.opencv_imgproc.contourArea(Lorg/bytedeco/opencv/opencv_core/Mat;Z)D (0 bytes) @ 0x00000001101edec4 [0x00000001101ede00+0xc4]`

[saudet]

Please try to use PointerScope: http://bytedeco.org/news/2018/07/17/bytedeco-as-distribution/

[iversionpeng]

Hello, I would like to ask: Both Mat and MatVector inherit Pointer. There is DeallocatorReference in the Point object. This is to save the memory address. During use, GC occurs, causing the previously created Mat memory to be recycled. Later use of this mat will cause the program to crash. This design Is it reasonable?

[saudet]

Please try to set the "org.bytedeco.javacpp.nopointergc" system property to "true".

[iversionpeng]

Please try to set the "org.bytedeco.javacpp.nopointergc" system property to "true".

We want to understand how javaCp manages off-heap memory so that we can use it more accurately. Can you provide some information for reference?

[iversionpeng]

Please try to set the "org.bytedeco.javacpp.nopointergc" system property to "true".

It is true that there will be no crash when using this method; but why might there be a crash when closinging it? In single thread case

[saudet]

If you need reliability, please don't use GC, use PointerScope. It works just like C++, that's pretty much all you need to know.

[iversionpeng]

If you need reliability, please don't use GC, use PointerScope. It works just like C++, that's pretty much all you need to know.

Thanks a lot!

It is the default way to release memory, Why is the GC method unreliable?

[saudet]

Because it's not designed to track anything else than heap memory.

[iversionpeng]

Because it's not designed to track anything else than heap memory.

Sorry to have bothered you for so long. I try to understand what you mean.

 MatVector contours = new MatVector();
 Mat hierarchy = new Mat();
 findContours(
             securityErodeMat,
             contours,
             hierarchy,
             RETR_CCOMP,
             CHAIN_APPROX_SIMPLE
     );
  Mat[] mats = contours.get();
  for (int i = 0; i < mats.length; ++i) {
         Mat myMat = collect.get(i);
        for (int y = 0; y < 800 * 1600; y++) {
             Scalar currentColor = new Scalar(h, s, v, 0);
 }
            double area = contourArea(myMat, false);
}

When GC is triggered, the DeallocatorReference in MatVector may be recycled. When I use Mat later in contourArea, may I get the wrong data?

[saudet]

Yes, that's possible

[iversionpeng]

Yes, that's possible Thanks a lot! Another question:

Where can I get its source code? Not by decompiling the case

[saudet]

You can ignore that library, it's not usually needed.

[iversionpeng]

You can ignore that library, it's not usually needed.

I am verifying the unreliability of relying on Gc to recycle off-heap memory; through the code, I cannot infer what the problem is with relying on GC to recycle off-heap memory, but the phenomenon is that I am using the MatVector object, and then there is a GC log and a Collecting log. Immediately afterwards, the memory space in MatVector was cleared; the code to clean up the off-heap memory relies on GC to trigger MatVector recycling. MatVector is a strong reference and it should not be recycled when I use it, so I feel that it is not triggered by GC, maybe It is the clear method called elsewhere, so I would like to see its underlying implementation; I would like to hear your opinions.

[saudet]

So, what you probably want is the JNI code used for OpenCV and that gets generated at build time: https://github.com/bytedeco/javacpp-presets#build-instructions