pyorbbecsdk
pyorbbecsdk copied to clipboard
orbbec
关于深度图分辨率和深度后处理API的问题
我使用的是Gemini336L，win10系统。然后想自定义RGB和深度图的尺寸，并对深度图进行自定义后处理。现在有三个问题：1. 我在设置了深度尺寸后，得到的数据分辨率不符合预定义。2. 我未在example中找到后处理的API，仅在post_processing中找到后处理的显示内容。3.我在使用Viewer.exe设置相机后，无法保存设置的参数(或者能让我在代码里配置参数吗)。代码如下：
两个分辨率都是 (w1280，h800)，我debug过，参数没问题。
`def start_ppl(edict_args): pipeline = Pipeline() config = Config() try: profile_list = pipeline.get_stream_profile_list(OBSensorType.COLOR_SENSOR) color_profile = profile_list.get_video_stream_profile(edict_args.width, edict_args.height, OBFormat.RGB, edict_args.fps) # 此处RGB设置的分辨率在后续color_image的shape是对的 config.enable_stream(color_profile) profile_list = pipeline.get_stream_profile_list(OBSensorType.DEPTH_SENSOR) depth_profile = profile_list.get_video_stream_profile(edict_args.width, edict_args.height, OBFormat.Y16, edict_args.fps) # 此处深度图设置的分辨率与后续depth_data的shape不同。
config.enable_stream(depth_profile)
except Exception as e:
print(e)
return
if edict_args.enable_sync:
try:
pipeline.enable_frame_sync()
except Exception as e:
print(e)
try:
pipeline.start(config)
except Exception as e:
print(e)
return
align_filter = AlignFilter(align_to_stream=OBStreamType.COLOR_STREAM) # align to color stream
while True:
try:
frames = pipeline.wait_for_frames(100)
if not frames:
continue
color_frame = frames.get_color_frame()
depth_frame = frames.get_depth_frame()
if not color_frame or not depth_frame:
continue
frames = align_filter.process(frames)
if not frames:
continue
frames = frames.as_frame_set()
color_frame = frames.get_color_frame()
depth_frame = frames.get_depth_frame()
if not color_frame or not depth_frame:
continue
color_image = frame_to_bgr_image(color_frame) # 数据shape符合预定义
if color_image is None:
print("Failed to convert frame to image")
continue
try:
device = pipeline.get_device()
assert device is not None
depth_sensor = device.get_sensor(OBSensorType.DEPTH_SENSOR)
assert depth_sensor is not None
filter_list = depth_sensor.get_recommended_filters()
assert filter_list is not None
# print filter list
for i in range(len(filter_list)): # 这里能拿出各种后处理类型，让我自己设置参数和组合吗？
post_filter = filter_list[i]
if post_filter:
print("filter name: ", post_filter.get_name())
print("filter is enabled: ", post_filter.is_enabled())
depth_data = depth_frame.get_data().astype(np.uint16).reshape(
(depth_frame.get_width(), depth_frame.get_height())) # depth_frame.get_data().astype(np.uint16)是一维的，且不等于预定义的h*w
except ValueError:
print("Failed to reshape depth data")
continue
depth_data = depth_data.astype(np.float32) * depth_frame.get_depth_scale()
depth_data = np.where((depth_data > edict_args.MIN_DEPTH) & (depth_data < edict_args.MAX_DEPTH), depth_data, 0)
depth_data = depth_data.astype(np.uint16)
depth_image = cv2.normalize(depth_data, None, 0, 255, cv2.NORM_MINMAX)
depth_image = cv2.applyColorMap(depth_image.astype(np.uint8), cv2.COLORMAP_JET)
depth_image = cv2.addWeighted(color_image, 0.5, depth_image, 0.5, 0)
`
I Modify sync_align.py ,doc refer to https://orbbec.github.io/pyorbbecsdk/source/4_Application_Guide/Data_Stream_acquisition.html#obtain-video-stream
# ******************************************************************************
# Copyright (c) 2024 Orbbec 3D Technology, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:# www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ******************************************************************************
import argparse
import sys
import cv2
import numpy as np
from pyorbbecsdk import *
from utils import frame_to_bgr_image
ESC_KEY = 27
MIN_DEPTH = 20 # 20mm
MAX_DEPTH = 10000 # 10000mm
# Temporal filter for smoothing depth data over time
def main(argv):
pipeline = Pipeline()
config = Config()
parser = argparse.ArgumentParser()
parser.add_argument("-s", "--enable_sync", help="enable sync", type=bool, default=True)
args = parser.parse_args(argv)
enable_sync = args.enable_sync
try:
profile_list = pipeline.get_stream_profile_list(OBSensorType.COLOR_SENSOR)
color_profile = profile_list.get_video_stream_profile(848, 480, OBFormat.RGB, 30)
config.enable_stream(color_profile)
profile_list = pipeline.get_stream_profile_list(OBSensorType.DEPTH_SENSOR)
depth_profile = profile_list.get_video_stream_profile(848, 480, OBFormat.Y16, 30)
config.enable_stream(depth_profile)
except Exception as e:
print(e)
return
if enable_sync:
try:
pipeline.enable_frame_sync()
except Exception as e:
print(e)
try:
pipeline.start(config)
except Exception as e:
print(e)
return
align_filter = AlignFilter(align_to_stream=OBStreamType.COLOR_STREAM)
while True:
try:
frames = pipeline.wait_for_frames(100)
if not frames:
continue
color_frame = frames.get_color_frame()
depth_frame = frames.get_depth_frame()
if not color_frame or not depth_frame:
continue
depth_width = depth_frame.get_width()
depth_height = depth_frame.get_height()
color_width = color_frame.get_width()
color_height = color_frame.get_height()
print(f"Depth Width: {depth_width},Depth Height: {depth_height}, Color Width: {color_width},Color Height: {color_height}")
frames = align_filter.process(frames)
if not frames:
continue
frames = frames.as_frame_set()
color_frame = frames.get_color_frame()
depth_frame = frames.get_depth_frame()
if not color_frame or not depth_frame:
continue
color_image = frame_to_bgr_image(color_frame)
if color_image is None:
print("Failed to convert frame to image")
continue
try:
depth_data = np.frombuffer(depth_frame.get_data(), dtype=np.uint16).reshape(
(depth_frame.get_height(), depth_frame.get_width()))
except ValueError:
print("Failed to reshape depth data")
continue
depth_data = depth_data.astype(np.float32) * depth_frame.get_depth_scale()
depth_data = np.where((depth_data > MIN_DEPTH) & (depth_data < MAX_DEPTH), depth_data, 0)
depth_data = depth_data.astype(np.uint16)
depth_image = cv2.normalize(depth_data, None, 0, 255, cv2.NORM_MINMAX)
depth_image = cv2.applyColorMap(depth_image.astype(np.uint8), cv2.COLORMAP_JET)
depth_image = cv2.addWeighted(color_image, 0.5, depth_image, 0.5, 0)
cv2.imshow("SyncAlignViewer", depth_image)
if cv2.waitKey(1) in [ord('q'), ESC_KEY]:
break
except KeyboardInterrupt:
break
cv2.destroyAllWindows()
pipeline.stop()
if name == "main": main(sys.argv[1:])
@zhonghong322
******************************************************************************
Copyright (c) 2024 Orbbec 3D Technology, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http:# www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
******************************************************************************
import argparse import sys
import cv2 import numpy as np
from pyorbbecsdk import * from utils import frame_to_bgr_image
ESC_KEY = 27
MIN_DEPTH = 20 # 20mm MAX_DEPTH = 10000 # 10000mm
Temporal filter for smoothing depth data over time
def main(argv): pipeline = Pipeline() config = Config() parser = argparse.ArgumentParser() parser.add_argument("-s", "--enable_sync", help="enable sync", type=bool, default=True) args = parser.parse_args(argv)
enable_sync = args.enable_sync
try:
profile_list = pipeline.get_stream_profile_list(OBSensorType.COLOR_SENSOR)
color_profile = profile_list.get_video_stream_profile(848, 480, OBFormat.RGB, 30)
config.enable_stream(color_profile)
profile_list = pipeline.get_stream_profile_list(OBSensorType.DEPTH_SENSOR)
depth_profile = profile_list.get_video_stream_profile(848, 480, OBFormat.Y16, 30)
config.enable_stream(depth_profile)
except Exception as e:
print(e)
return
if enable_sync:
try:
pipeline.enable_frame_sync()
except Exception as e:
print(e)
try:
pipeline.start(config)
except Exception as e:
print(e)
return
align_filter = AlignFilter(align_to_stream=OBStreamType.COLOR_STREAM)
while True:
try:
frames = pipeline.wait_for_frames(100)
if not frames:
continue
color_frame = frames.get_color_frame()
depth_frame = frames.get_depth_frame()
if not color_frame or not depth_frame:
continue
depth_width = depth_frame.get_width()
depth_height = depth_frame.get_height()
color_width = color_frame.get_width()
color_height = color_frame.get_height()
print(f"Depth Width: {depth_width},Depth Height: {depth_height}, Color Width: {color_width},Color Height: {color_height}")
frames = align_filter.process(frames)
if not frames:
continue
frames = frames.as_frame_set()
color_frame = frames.get_color_frame()
depth_frame = frames.get_depth_frame()
if not color_frame or not depth_frame:
continue
color_image = frame_to_bgr_image(color_frame)
if color_image is None:
print("Failed to convert frame to image")
continue
try:
depth_data = np.frombuffer(depth_frame.get_data(), dtype=np.uint16).reshape(
(depth_frame.get_height(), depth_frame.get_width()))
except ValueError:
print("Failed to reshape depth data")
continue
depth_data = depth_data.astype(np.float32) * depth_frame.get_depth_scale()
depth_data = np.where((depth_data > MIN_DEPTH) & (depth_data < MAX_DEPTH), depth_data, 0)
depth_data = depth_data.astype(np.uint16)
depth_image = cv2.normalize(depth_data, None, 0, 255, cv2.NORM_MINMAX)
depth_image = cv2.applyColorMap(depth_image.astype(np.uint8), cv2.COLORMAP_JET)
depth_image = cv2.addWeighted(color_image, 0.5, depth_image, 0.5, 0)
cv2.imshow("SyncAlignViewer", depth_image)
if cv2.waitKey(1) in [ord('q'), ESC_KEY]:
break
except KeyboardInterrupt:
break
cv2.destroyAllWindows()
pipeline.stop()
有两个问题请问一下：1. 该代码能否在空间坐标上将深度和RGB对齐？(如将Depth对齐到RGB上)。2. 该代码能否在时间上对齐同时间戳的RGB和Depth呢？(此外，如何获取时间戳呀)