Assistance Needed for Streaming ZED Camera Video to Web Browser Using GStreamer

General Discussion
1

Hello GStreamer forums,

I’m a new member and would appreciate your assistance with the following code:

import sys
import numpy as np
import pyzed.sl as sl
import cv2
import gi
gi.require_version(‘Gst’, ‘1.0’)
gi.require_version(‘GstRtspServer’, ‘1.0’)
from gi.repository import Gst, GstRtspServer, GObject

def main():
# Create a ZED camera object
zed = sl.Camera()

init = sl.InitParameters()
init.camera_resolution = sl.RESOLUTION.HD720
init.camera_fps = 30
init.set_from_serial_number(serial)

# Open the camera
err = zed.open(init)
if err != sl.ERROR_CODE.SUCCESS:
    print(repr(err))
    zed.close()
    exit(1)

# Set runtime parameters
runtime = sl.RuntimeParameters()

# Image size
image_size = sl.Resolution(1280, 720)
image_size_out = sl.Resolution(1280, 720)

# Create sl.Mat matrices
image_zed = sl.Mat(image_size.width, image_size.height, sl.MAT_TYPE.U8_C4)
image_zed_out = sl.Mat(image_size_out.width, image_size_out.height, sl.MAT_TYPE.U8_C4)

# Configure GStreamer pipeline
out_send = cv2.VideoWriter(
    'appsrc is-live=false ! videoconvert ! video/x-raw,format=YUY2,width=1280,height=720,framerate=30/1 ! '
    'nvvidconv ! nvv4l2h264enc insert-sps-pps=1 bitrate=9000000 ! video/x-h264, '
    'stream-format=byte-stream ! rtph264pay config-interval=1 ! '
    'udpsink host=(ip) port=3000',
    cv2.CAP_GSTREAMER, 0, 30, (1280, 720), True)

if not out_send.isOpened():
    print('VideoWriter not opened')
    exit(0)

rtsp_port_num = 3000

# Set up RTSP server
server = GstRtspServer.RTSPServer.new()
server.props.service = "%d" % rtsp_port_num
server.attach(None)

codec = "H264"
updsink_port_num = 3000
factory = GstRtspServer.RTSPMediaFactory.new()
factory.set_launch(
    "(udpsrc name=pay0 port=%d buffer-size=524288 caps=\"application/x-rtp, media=video, clock-rate=90000, encoding-name=(string)%s, payload=96 \" )"
    % (updsink_port_num, codec))

factory.set_shared(True)
server.get_mount_points().add_factory("/ds-test", factory)

# Output RTSP info
print("\n *** Launched RTSP Streaming at rtsp://localhost:%d/ds-test ***\n\n" % rtsp_port_num)

key = ' '
while True:
    err = zed.grab(runtime)

    if err == sl.ERROR_CODE.SUCCESS:
        # Retrieve images
        zed.retrieve_image(image_zed, sl.VIEW.LEFT, sl.MEM.CPU, image_size)
        zed.retrieve_image(image_zed_out, sl.VIEW.LEFT)

        # Convert sl.Mat to numpy array for OpenCV
        image_ocv = image_zed.get_data()
        image_ocv_out = image_zed_out.get_data()

        # Convert RGBA to RGB
        image_RGB = cv2.cvtColor(image_ocv_out, cv2.COLOR_RGBA2RGB)

        # Stream using RTSP
        out_send.write(image_RGB)
        key = cv2.waitKey(1)

cv2.destroyAllWindows()
zed.close()

if name == “main”:
main()

I am able to stream the video locally using VLC on another machine over the local network, and the stream works well with a small delay that is currently acceptable. However, my main concern is finding a way to stream the video to a web browser without relying on VLC. I have tried various methods without success, and I would greatly appreciate your guidance to achieve this.

Additional Information:

  1. I need to use the ZED SDK to leverage its depth mapping and other features, so removing it from the code is not an option.
  2. Due to bandwidth limitations, I must use high-efficiency encoding such as H.264.

Thank you in advance for your support!