jdibenes
Simultaniusly get pv stream with and without Holograms
Dear author,
Thank you very much for providing this real-time streaming. Currently I am using the pv stream to detect objects and use the Holograms to visualize the animations. However, I cannot get pv stream with and without HOLOgrams at the same time. If I use pv stream of Holograms, then the virtual objects are detected instead of the real objects. If I use pv stream without Holograms, then I cannot see the animations.
Is it possible to get pv stream with and without Holograms at the same time?
Best regards
Hi, Try using PV with holograms along with Extended Video in shared mode with PV selected to obtain a second PV stream without holograms. See client_stream_extended_video.py.
Hello ! First, thank you very much for your hard work on this library, it is very usefull :)
I tried to stream simultaneously two PV (one with MRC the other without it) according to the code on client_stream_extended_video.py and the multiprocessing of sample_video.py but it did not work...
Here is my code :
import multiprocessing as mp
from pynput import keyboard
import cv2
import hl2ss
import hl2ss_lnm
import hl2ss_mp
HOST = "192.168.1.7"
CAMERA_WIDTH = 960
CAMERA_HEIGHT = 540
CAMERA_FPS = 30
USE_EXTENDED_VIDEO = True
if __name__ == "__main__":
enable = True
def on_press(key):
global enable
enable = key != keyboard.Key.esc
return enable
listener = keyboard.Listener(on_press=on_press)
listener.start()
hl2ss_lnm.start_subsystem_pv(
HOST,
hl2ss.StreamPort.PERSONAL_VIDEO,
enable_mrc=True,
)
if USE_EXTENDED_VIDEO:
hl2ss_lnm.start_subsystem_pv(
HOST,
hl2ss.StreamPort.EXTENDED_VIDEO,
enable_mrc=False,
global_opacity=0,
output_width=2,
output_height=4,
shared=True,
)
# Buffer length in seconds
buffer_length = 5
producer = hl2ss_mp.producer()
consumer = hl2ss_mp.consumer()
manager = mp.Manager()
producer.configure(
hl2ss.StreamPort.PERSONAL_VIDEO,
hl2ss_lnm.rx_pv(
HOST,
hl2ss.StreamPort.PERSONAL_VIDEO,
mode=hl2ss.StreamMode.MODE_1,
width=CAMERA_WIDTH,
height=CAMERA_HEIGHT,
framerate=CAMERA_FPS,
divisor=1,
profile=hl2ss.VideoProfile.H265_MAIN,
bitrate=None,
decoded_format="bgr24",
),
)
producer.initialize(
hl2ss.StreamPort.PERSONAL_VIDEO, CAMERA_FPS * buffer_length
)
producer.start(hl2ss.StreamPort.PERSONAL_VIDEO)
sink_pv: hl2ss_mp._sink = consumer.create_sink(
producer, hl2ss.StreamPort.PERSONAL_VIDEO, manager, ...
) # see create_sink method for "..."
sink_pv.get_attach_response()
if USE_EXTENDED_VIDEO:
producer.configure(
hl2ss.StreamPort.EXTENDED_VIDEO,
hl2ss_lnm.rx_pv(
HOST,
hl2ss.StreamPort.EXTENDED_VIDEO,
mode=hl2ss.StreamMode.MODE_0,
width=CAMERA_WIDTH,
height=CAMERA_HEIGHT,
framerate=CAMERA_FPS,
divisor=1,
profile=hl2ss.VideoProfile.H265_MAIN,
bitrate=None,
decoded_format="bgr24",
),
)
producer.initialize(
hl2ss.StreamPort.EXTENDED_VIDEO, CAMERA_FPS * buffer_length
)
producer.start(hl2ss.StreamPort.EXTENDED_VIDEO)
sink_ev: hl2ss_mp._sink = consumer.create_sink(
producer, hl2ss.StreamPort.EXTENDED_VIDEO, manager, ...
) # see create_sink method for "..."
sink_ev.get_attach_response()
while(enable):
sink_pv.acquire()
_, data_pv = sink_pv.get_most_recent_frame()
if data_pv is None:
continue
if USE_EXTENDED_VIDEO:
sink_ev.acquire()
_, data_ev = sink_ev.get_nearest(data_pv.timestamp)
if data_ev is None:
print("No data")
extended_video_OK = False
else:
extended_video_OK = True
pv_img = data_pv.payload.image
cv2.imshow("PV", pv_img)
if USE_EXTENDED_VIDEO and extended_video_OK:
ev_img = data_ev.payload.image
cv2.imshow("EV", ev_img)
cv2.waitKey(1)
sink_pv.detach()
producer.stop(hl2ss.StreamPort.PERSONAL_VIDEO)
if USE_EXTENDED_VIDEO:
sink_ev.detach()
producer.stop(hl2ss.StreamPort.EXTENDED_VIDEO)
hl2ss_lnm.stop_subsystem_pv(HOST, hl2ss.StreamPort.PERSONAL_VIDEO)
if USE_EXTENDED_VIDEO:
hl2ss_lnm.stop_subsystem_pv(HOST, hl2ss.StreamPort.EXTENDED_VIDEO)
When USE_EXTENDED_VIDEO is False, the PV Video works just fine, but when it is True the code is blocked at sink_ev.acquire() with no error raised.
Thank you again, best regards
Hi, When PV and EV are used to obtain two streams of the front camera, the streams must be initialized sequentially. I didn't document this, my bad. Here is the modified script:
import multiprocessing as mp
from pynput import keyboard
import cv2
import hl2ss
import hl2ss_lnm
import hl2ss_mp
HOST = "192.168.1.7"
CAMERA_WIDTH = 960
CAMERA_HEIGHT = 540
CAMERA_FPS = 30
USE_EXTENDED_VIDEO = True
if __name__ == "__main__":
enable = True
def on_press(key):
global enable
enable = key != keyboard.Key.esc
return enable
listener = keyboard.Listener(on_press=on_press)
listener.start()
hl2ss_lnm.start_subsystem_pv(
HOST,
hl2ss.StreamPort.PERSONAL_VIDEO,
enable_mrc=True,
)
# Buffer length in seconds
buffer_length = 5
producer = hl2ss_mp.producer()
consumer = hl2ss_mp.consumer()
manager = mp.Manager()
producer.configure(
hl2ss.StreamPort.PERSONAL_VIDEO,
hl2ss_lnm.rx_pv(
HOST,
hl2ss.StreamPort.PERSONAL_VIDEO,
mode=hl2ss.StreamMode.MODE_1,
width=CAMERA_WIDTH,
height=CAMERA_HEIGHT,
framerate=CAMERA_FPS,
divisor=1,
profile=hl2ss.VideoProfile.H265_MAIN,
bitrate=None,
decoded_format="bgr24",
),
)
producer.initialize(
hl2ss.StreamPort.PERSONAL_VIDEO, CAMERA_FPS * buffer_length
)
producer.start(hl2ss.StreamPort.PERSONAL_VIDEO)
sink_pv: hl2ss_mp._sink = consumer.create_sink(
producer, hl2ss.StreamPort.PERSONAL_VIDEO, manager, ...
) # see create_sink method for "..."
sink_pv.get_attach_response()
if USE_EXTENDED_VIDEO:
# Since PV and EV use the same camera in this script,
# start EV shared after PV configuration on the server is complete,
# so start EV after the first PV frame is received
sink_pv.acquire()
hl2ss_lnm.start_subsystem_pv(
HOST,
hl2ss.StreamPort.EXTENDED_VIDEO,
enable_mrc=False,
global_opacity=0,
output_width=2,
output_height=4,
shared=True,
)
producer.configure(
hl2ss.StreamPort.EXTENDED_VIDEO,
hl2ss_lnm.rx_pv(
HOST,
hl2ss.StreamPort.EXTENDED_VIDEO,
mode=hl2ss.StreamMode.MODE_0,
width=CAMERA_WIDTH,
height=CAMERA_HEIGHT,
framerate=CAMERA_FPS,
divisor=1,
profile=hl2ss.VideoProfile.H265_MAIN,
bitrate=None,
decoded_format="bgr24",
),
)
producer.initialize(
hl2ss.StreamPort.EXTENDED_VIDEO, CAMERA_FPS * buffer_length
)
producer.start(hl2ss.StreamPort.EXTENDED_VIDEO)
sink_ev: hl2ss_mp._sink = consumer.create_sink(
producer, hl2ss.StreamPort.EXTENDED_VIDEO, manager, None
) # see create_sink method for "..."
sink_ev.get_attach_response()
while(enable):
sink_pv.acquire()
_, data_pv = sink_pv.get_most_recent_frame()
if data_pv is None:
continue
if USE_EXTENDED_VIDEO:
_, data_ev = sink_ev.get_nearest(data_pv.timestamp)
if data_ev is None:
print("No data")
extended_video_OK = False
else:
extended_video_OK = True
pv_img = data_pv.payload.image
cv2.imshow("PV", pv_img)
if USE_EXTENDED_VIDEO and extended_video_OK:
ev_img = data_ev.payload.image
cv2.imshow("EV", ev_img)
cv2.waitKey(1)
sink_pv.detach()
producer.stop(hl2ss.StreamPort.PERSONAL_VIDEO)
if USE_EXTENDED_VIDEO:
sink_ev.detach()
producer.stop(hl2ss.StreamPort.EXTENDED_VIDEO)
hl2ss_lnm.stop_subsystem_pv(HOST, hl2ss.StreamPort.PERSONAL_VIDEO)
if USE_EXTENDED_VIDEO:
hl2ss_lnm.stop_subsystem_pv(HOST, hl2ss.StreamPort.EXTENDED_VIDEO)