depthai
depthai copied to clipboard
luxonis
[BUG] FFC-type Board Calibration Flashing Error
Describe the bug
I am on latest commit 39da981db8b886cf6c88b55a044724a82619a794 (HEAD -> multi_cam_support) @ depthai-python repo, and tries to flash calibration parameters into a SoM unit. I currently test on a OAK-FFC-3P board.
I am trying to flash the extrinsic and intrinsic paramters of camera 0 and 1 (rgb and left). I encounter the error of Extrinsics between all the cameras is not found with single head and a tail, but in actual fact, the chain does exist in the original .json file, by manual inspection.
However, if i exchange the definition of 0 and 1 slots in the calibration .json file, the flashing works. This appears to be a bug. How can swaping the "chain" direction, affect the sanity check?
To Reproduce Steps to reproduce the behavior:
- Use
calibration_flash.pyscript to try to flash the following to calibration json files - The first one shows error, but the second one passes
Expected behavior
In both cases of .json files, the calibration flashing should pass (or fail together)
Attach system log
[2022-08-18 16:48:10.779] [debug] Extrinsics between all the cameras is not found with single head and a tail
[2022-08-18 16:48:10.779] [debug] Found link to a CameraID whose camera calibration is not loaded. Please cross check the connection by creating a json file using eepromToJsonFile().
[2022-08-18 16:48:10.779] [debug] Device about to be closed...
[2022-08-18 16:48:11.513] [debug] Watchdog thread exception caught: Couldn't write data to stream: '__watchdog' (X_LINK_ERROR)
[2022-08-18 16:48:11.637] [debug] Timesync thread exception caught: Couldn't read data from stream: '__timesync' (X_LINK_ERROR)
[2022-08-18 16:48:11.637] [debug] Log thread exception caught: Couldn't read data from stream: '__log' (X_LINK_ERROR)
[2022-08-18 16:48:11.638] [debug] XLinkResetRemote of linkId: (0)
[2022-08-18 16:48:11.638] [debug] Device closed, 858
Traceback (most recent call last):
File "calibration_flash.py", line 24, in <module>
status = device.flashCalibration(calibData)
RuntimeError: Failed to validate the extrinsics connection. Enable debug mode for more information.
Additional context Discord discussion here https://discord.com/channels/790680891252932659/1004761233104322650
The file that would gives error flashError.json
{
"batchName": "",
"batchTime": 0,
"boardConf": "",
"boardCustom": "",
"boardName": "",
"boardOptions": 0,
"boardRev": "",
"cameraData": [
[
1,
{
"cameraType": 1,
"distortionCoeff": [
-0.09709959477186203,
-0.010917834006249905,
0.013512088917195797,
-0.00451677106320858,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"extrinsics": {
"rotationMatrix": [],
"specTranslation": {
"x": 0.0,
"y": 0.0,
"z": 0.0
},
"toCameraSocket": -1,
"translation": {
"x": 0.0,
"y": 0.0,
"z": 0.0
}
},
"height": 800,
"intrinsicMatrix": [
[
714.5363159179688,
0.0,
656.96337890625
],
[
0.0,
713.7963256835938,
424.70001220703125
],
[
0.0,
0.0,
1.0
]
],
"lensPosition": 0,
"specHfovDeg": 105.0,
"width": 1280
}
],
[
0,
{
"cameraType": 1,
"distortionCoeff": [
-0.09991353750228882,
0.00044088903814554214,
0.001793952425941825,
-0.0008543016738258302,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"extrinsics": {
"rotationMatrix": [
[
0.9998628497123718,
-0.01594027690589428,
-0.004497510846704245
],
[
0.015798956155776978,
0.9994286894798279,
-0.029878731817007065
],
[
0.004971216432750225,
0.029803577810525894,
0.9995434284210205
]
],
"specTranslation": {
"x": -5.579999923706055,
"y": 0.0,
"z": 0.0
},
"toCameraSocket": 1,
"translation": {
"x": -5.531078815460205,
"y": -0.04055831953883171,
"z": 0.016503548249602318
}
},
"height": 800,
"intrinsicMatrix": [
[
710.3955078125,
0.0,
626.0454711914063
],
[
0.0,
709.4725952148438,
383.04241943359375
],
[
0.0,
0.0,
1.0
]
],
"lensPosition": 0,
"specHfovDeg": 105.0,
"width": 1280
}
]
],
"hardwareConf": "",
"imuExtrinsics": {
"rotationMatrix": [],
"specTranslation": {
"x": 0.0,
"y": 0.0,
"z": 0.0
},
"toCameraSocket": -1,
"translation": {
"x": 0.0,
"y": 0.0,
"z": 0.0
}
},
"miscellaneousData": [],
"productName": "",
"stereoRectificationData": {
"leftCameraSocket": -1,
"rectifiedRotationLeft": [],
"rectifiedRotationRight": [],
"rightCameraSocket": -1
},
"version": 7
}
A passing json file, with only the camera slots number 0 and 1 swapped. flashOK.json
{
"batchName": "",
"batchTime": 0,
"boardConf": "",
"boardCustom": "",
"boardName": "",
"boardOptions": 0,
"boardRev": "",
"cameraData": [
[
0,
{
"cameraType": 1,
"distortionCoeff": [
-0.09709959477186203,
-0.010917834006249905,
0.013512088917195797,
-0.00451677106320858,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"extrinsics": {
"rotationMatrix": [],
"specTranslation": {
"x": 0.0,
"y": 0.0,
"z": 0.0
},
"toCameraSocket": -1,
"translation": {
"x": 0.0,
"y": 0.0,
"z": 0.0
}
},
"height": 800,
"intrinsicMatrix": [
[
714.5363159179688,
0.0,
656.96337890625
],
[
0.0,
713.7963256835938,
424.70001220703125
],
[
0.0,
0.0,
1.0
]
],
"lensPosition": 0,
"specHfovDeg": 105.0,
"width": 1280
}
],
[
1,
{
"cameraType": 1,
"distortionCoeff": [
-0.09991353750228882,
0.00044088903814554214,
0.001793952425941825,
-0.0008543016738258302,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"extrinsics": {
"rotationMatrix": [
[
0.9998628497123718,
-0.01594027690589428,
-0.004497510846704245
],
[
0.015798956155776978,
0.9994286894798279,
-0.029878731817007065
],
[
0.004971216432750225,
0.029803577810525894,
0.9995434284210205
]
],
"specTranslation": {
"x": -5.579999923706055,
"y": 0.0,
"z": 0.0
},
"toCameraSocket": 0,
"translation": {
"x": -5.531078815460205,
"y": -0.04055831953883171,
"z": 0.016503548249602318
}
},
"height": 800,
"intrinsicMatrix": [
[
710.3955078125,
0.0,
626.0454711914063
],
[
0.0,
709.4725952148438,
383.04241943359375
],
[
0.0,
0.0,
1.0
]
],
"lensPosition": 0,
"specHfovDeg": 105.0,
"width": 1280
}
]
],
"hardwareConf": "",
"imuExtrinsics": {
"rotationMatrix": [],
"specTranslation": {
"x": 0.0,
"y": 0.0,
"z": 0.0
},
"toCameraSocket": -1,
"translation": {
"x": 0.0,
"y": 0.0,
"z": 0.0
}
},
"miscellaneousData": [],
"productName": "",
"stereoRectificationData": {
"leftCameraSocket": -1,
"rectifiedRotationLeft": [],
"rectifiedRotationRight": [],
"rightCameraSocket": -1
},
"version": 7
}
@chengguizi Thank you for the bug report and details! Will investigate more today/tomorrow, sorry for delay.
@chengguizi yes, I get the same error with your flashFail example
Failed to validate the extrinsics connection. Enable debug mode for more information.
And it works with the other one. I'll try to investigate more, how did you generated the json with the error?
Hi @daxoft , I wrote a little python script to convert the results from a calibration tool to depthai format.
The generation of the json is done by the API calibData = dai.CalibrationHandler(), necessary API calls, and finally calibData.eepromToJsonFile("temp.json").
I cant share the full one, but this is the main code logic:
with dai.Device() as device:
calibData = dai.CalibrationHandler()
# calibData.setBoardInfo("BW1098OBC", "R0M0E0")
# calibData.setBoardInfo("OAK-D-LITE", "R0M0E0")
# eepromData = calibData.getEepromData()
# print(eepromData.boardName)
# print(eepromData.boardRev)
# print(eepromData.cameraData)
# print(eepromData.imuExtrinsics)
# print(eepromData.stereoRectificationData)
# print(eepromData.version)
# exit(0)
with open(args.JsonFile, 'r') as stream:
try:
Json = json.load(stream)['value0']
except:
print("Error occurred for json")
exit(1)
leftId = dai.CameraBoardSocket.LEFT # 1
rightId = dai.CameraBoardSocket.RGB # 2
# STEP 1 - intrinsic
leftCameraType = Json['intrinsics'][0]['camera_type']
rightCameraType = Json['intrinsics'][1]['camera_type']
leftIntrinsic = getIntrinsicMatrix(Json['intrinsics'][0]['intrinsics'])
rightIntrinsic = getIntrinsicMatrix(Json['intrinsics'][1]['intrinsics'])
print("\nIntrinsic Matrices")
print(leftIntrinsic)
print(rightIntrinsic)
leftResolution = Json['resolution'][0]
rightResolution = Json['resolution'][1]
print('\nResolutions')
print(leftResolution)
print(rightResolution)
calibData.setCameraIntrinsics(leftId, leftIntrinsic.tolist(), leftResolution[0], leftResolution[1])
calibData.setCameraIntrinsics(rightId, rightIntrinsic.tolist(), rightResolution[0], rightResolution[1])
# STEP 2 - distortion
# calibData.setCameraType(leftId, dai.CameraModel.Perspective)
# calibData.setCameraType(rightId, dai.CameraModel.Perspective)
leftD = getDistortionCoeffsFisheye(Json['intrinsics'][0]['intrinsics'])
rightD = getDistortionCoeffsFisheye(Json['intrinsics'][1]['intrinsics'])
print(f"\nDistortion Coeffs ({leftCameraType}, {rightCameraType})")
print(leftD)
print(rightD)
calibData.setDistortionCoefficients(leftId, leftD.tolist())
calibData.setDistortionCoefficients(rightId, rightD.tolist())
if (leftCameraType == 'kb4' and rightCameraType == 'kb4'):
print('\nSelect Fisheye Camera Type')
calibData.setCameraType(leftId, dai.CameraModel.Fisheye)
calibData.setCameraType(rightId, dai.CameraModel.Fisheye)
else:
raise RuntimeError(f"unknown camera type {leftCameraType} {rightCameraType}")
# STEP 3 - extrinsic
T_imu_left = getExtrinsicTransform(Json['T_imu_cam'][0])
T_imu_right = getExtrinsicTransform(Json['T_imu_cam'][1])
print('\nExtrinsic Matrices')
print(T_imu_left.matrix())
print(T_imu_right.matrix())
right_T_left = T_imu_right.inverse() * T_imu_left
leftR, leftt = getRt(right_T_left.matrix())
print("\nLeft Camera Extrinsic")
print(leftR)
print(leftt)
# translation in cm
calibData.setCameraExtrinsics(leftId, rightId, leftR, leftt * 100, [-5.5,0,0])
# calibData.setFov(leftId, 105)
# calibData.setFov(rightId, 105)
# STEP 3A - imu extrinsic
# T_right_imu = T_imu_right.inverse()
# imuR, imut = getRt(T_right_imu.matrix())
# spec4CamParallel = [-2,0,-3.5]
# calibData.setImuExtrinsics(rightId, imuR, imut * 100, spec4CamParallel)
# STEP 4 - rectification
# The step 4 is not necessary, as we shall calculate online
# R1, R2, P1, P2, Q = cv2.fisheye.stereoRectify(leftIntrinsic, leftD[0:4], rightIntrinsic, rightD[0:4], (leftResolution[0], leftResolution[1]), leftR, leftt, cv2.CALIB_ZERO_DISPARITY)
# print(f"R1\n{R1}")
# print(f"R2\n{R2}")
# print(f"P1\n{P1}")
# print(f"P2\n{P2}")
# print(f"Q\n{Q}")
# calibData.setStereoLeft(leftId, R1.tolist())
# calibData.setStereoRight(rightId, R2.tolist())
status = device.flashCalibration(calibData)
if status:
print('Calibration Flash Successful')
else:
print('Calibration Flash Failed!!!')
@chengguizi sorry for delay, was caught with other tasks, will calibrate FFC-4P and FFC-3P tomorrow and will get back to you with new info.
Oh yes. Due to how our finished products were designed there was a constraints added to avoid linking of extrinsicis. Which was fixed in a recent PR.
Please check with this PR or on develop.
https://github.com/luxonis/depthai-python/pull/659
Sorry I didn't notice this issue ahead.
Thanks @saching13 ! Any chance this will go into multi_cam_support and other custom branches sometime?
will be added to multi_cam_support today. others gets updated when synced to develop