Overlaying rtsp feeds and streaming to web

Application Development
8

This time I’ve been able to further test your case.

1. I think that the main issue you’re facing is the management of the queue. I’m not so familiar with python and its Threads/Queue modules, but just adapting your code to do that in the process_frames loop should convince you:

import cv2
from flask import Flask, Response
from imutils.video import VideoStream
import time

app = Flask(__name__)

# Colours, dimensions and other settings
RED = (0, 0, 255)   # BGR

src_width = 1920
src_height = 1080

xstep = 5
xpos = range(0, src_width, xstep)
xnum = int(src_width / xstep)
ypos = src_height / 2


def process_frames():
    src_stream = VideoStream(src=src_url).start()

    while True:
        frame = src_stream.read()
        xposIdx = int(process_frames.frame_nb) % int(xnum)
        x = xpos[xposIdx]
        y = ypos
        x_end = x + 100  # Adjust the width of the box
        y_end = y + 100  # Adjust the height of the box

        # Draw red box on the video
        cv2.rectangle(frame, (int(x), int(y)), (int(x_end), int(y_end)), RED, 2)
            
        # Encode the frame as JPEG
        _, jpeg = cv2.imencode('.jpg', frame)
        frame_bytes = jpeg.tobytes()

        yield (b'--frame\r\n'
               b'Content-Type: image/jpeg\r\n\r\n' + frame_bytes + b'\r\n\r\n')
        process_frames.frame_nb = process_frames.frame_nb + 1
        #cv2.waitKey(1)
    src_stream.stop()
process_frames.frame_nb = 0

@app.route('/video_feed')
def video_feed():
    return Response(process_frames(), mimetype='multipart/x-mixed-replace; boundary=frame')  


def main():
    app.run(host='127.0.0.1', port=5560, debug=True)
    cv2.destroyAllWindows()

if __name__ == '__main__':
    src_url = "rtsp://127.0.0.1:8554/test"
    main()

And you may use one of the following pipelines for testing from localhost:

gst-launch-1.0 souphttpsrc is-live=1 location=http://127.0.0.1:5560/video_feed ! queue ! multipartdemux single-stream=1 ! image/jpeg ! jpegdec ! videoconvert ! autovideosink

# Or using NVDEC
gst-launch-1.0 souphttpsrc is-live=1 location=http://127.0.0.1:5560/video_feed ! queue ! multipartdemux single-stream=1 ! image/jpeg ! nvv4l2decoder mjpeg=1 ! nvvidconv ! autovideosink

One part of the missed synchro issue may also come from imutils.

2. Using gstreamer backend allows a much more periodic frame acquisition and processing. Also try:

import cv2
from flask import Flask, Response

# Check that your opencv build for this python version has gstreamer videoio support
import re
print('GStreamer support: %s' % re.search(r'GStreamer\:\s+(.*)', cv2.getBuildInformation()).group(1))


app = Flask(__name__)

# Colours, dimensions and other settings
RED = (0, 0, 255)   # BGR

def process_frames():
    gst_cap_str = 'uridecodebin uri=' + src_url + ' ! nvvidconv ! video/x-raw,format=BGRx ! videoconvert ! video/x-raw,format=BGR ! queue ! appsink drop=1'
    print(gst_cap_str)
    cap = cv2.VideoCapture(gst_cap_str, cv2.CAP_GSTREAMER)
    if not cap.isOpened():
        print('Failed to open capture. Exiting')
        quit()
    fps = float(cap.get(cv2.CAP_PROP_FPS))
    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
    size = (w, h)
    print(f"Size: {(size)}, FPS: {fps}")

    xstep = 5
    xpos = range(0, w, xstep)
    xnum = int(w / xstep)
    ypos = h / 2

    while True:
        ret, frame = cap.read()
        if not ret:
            print('Failed to read from camera, aborting')
            break
        xposIdx = int(process_frames.frame_nb) % xnum
        x = xpos[xposIdx]
        y = ypos
        x_end = x + 100  # Adjust the width of the box
        y_end = y + 100  # Adjust the height of the box

        # Draw red box on the video
        cv2.rectangle(frame, (int(x), int(y)), (int(x_end), int(y_end)), RED, 2)
            
        # Encode the frame as JPEG
        _, jpeg = cv2.imencode('.jpg', frame)
        frame_bytes = jpeg.tobytes()

        yield (b'--frame\r\n'
               b'Content-Type: image/jpeg\r\n\r\n' + frame_bytes + b'\r\n\r\n')
        process_frames.frame_nb = process_frames.frame_nb + 1
        #cv2.waitKey(1)
    cap.release()
process_frames.frame_nb = 0

@app.route('/video_feed')
def video_feed():
    return Response(process_frames(), mimetype='multipart/x-mixed-replace; boundary=frame')  


def main():
    app.run(host='127.0.0.1', port=5560, debug=True)
    cv2.destroyAllWindows()

if __name__ == '__main__':
    src_url = "rtsp://127.0.0.1:8554/test"
    main()

You can safely ignore the warnings about gstreamer backend unable to query duration and compute position.

1 Like