Hi
My setup and goal
I’m on Ubuntu Server minimal install using kmssink and multiple video streams being fed into a compositor, and their audio to an audiomixer.
I need to be able to switch which video is shown, perserve sync between videos, and do this without stutter/flash.
Issue
The current issue is, when I switch which video has alpha=1.0, sometimes a checkerboard pattern appears. I have confirmed this is from the compositor, by changing compositor background to black, this results in the same experience, sometimes a black screen appears during switch.
I had another attempt, where both videos always had alpha=1.0, but I instead just changed the zorder to define which video was shown. Here, I also saw the checkerboard pattern (I’m unsure if I tried having compositor background black for this approach).
I’m unsure what the norm is on these forums, but this is my code. I have tried setting GST_DEBUG to 4, but nothing was printed when the checkerboard pattern appeared.
My “controls” import is a custom script that handles keyboard inputs, I doubt it is relevant for this issue.
I had a previous version (AI generated, decided to drop it and actually learn GStreamer) that transitioned perfectly fine, but had stuttery playback during regular playback (not on switches). The code I pasted here, has perfect playback, but checkerboard when switching.
Any help is greatly appreciated.
import os
os.environ['GST_DEBUG'] = '0'
import gi
gi.require_version("Gst", "1.0")
from gi.repository import Gst, GLib
import time
from controls import create_input_handler, InputType
import logging
import sys
logging.basicConfig(level=logging.DEBUG, format="[%(name)s] [%(levelname)8s] - %(message)s")
logger = logging.getLogger(__name__)
Gst.init(None)
# THIS FILE HAS 2 .mp4 INPUTs and 2 .wav INPUTs. IT USES AUDIOMIXER. COMPOSITOR IS USED FOR THE VIDEO
# CONTROLS HAS BEEN ADDED, THEY SWITCH SCENE AND AUDIO
# TEST SUCCESS: NO STUTTER, CLEAN SWITCH, SOMETIMES CHECKERBOARD WHEN SWITCHING
class VideoHandler:
def __init__(self):
self.current_index = 0
self.v_source0 = Gst.ElementFactory.make("uridecodebin", "vsource0")
self.a_source0 = Gst.ElementFactory.make("uridecodebin", "asource0")
self.v_source1 = Gst.ElementFactory.make("uridecodebin", "vsource1")
self.a_source1 = Gst.ElementFactory.make("uridecodebin", "asource1")
self.v_compositor = Gst.ElementFactory.make("compositor", "compositor")
self.v_convert = Gst.ElementFactory.make("videoconvert", "videoconvert")
self.a_mixer = Gst.ElementFactory.make("audiomixer", "audiomixer")
self.a_convert = Gst.ElementFactory.make("audioconvert", "audioconvert")
self.a_resample = Gst.ElementFactory.make("audioresample", "audioresample")
self.a_sink = Gst.ElementFactory.make("autoaudiosink", "audiosink")
self.v_queue = Gst.ElementFactory.make("queue", "videoqueue")
self.v_queue.set_property("max-size-buffers", 0)
self.v_queue.set_property("max-size-bytes", 0)
self.v_queue.set_property("max-size-time", 80000000)
self.v_sink = Gst.ElementFactory.make("kmssink", "videosink")
self.v_sink.set_property("skip-vsync", True)
self.v_sink.set_property("sync", True)
self.v_sink.set_property("can-scale", False)
self.pipeline = Gst.Pipeline.new("switcher-pipeline")
if not self.v_source0 or not self.a_source0 or not self.v_source1 or not self.a_source1 or not self.a_mixer or not self.a_convert or not self.a_resample or not self.a_sink or not self.v_compositor or not self.v_convert or not self.v_queue or not self.v_sink:
logger.error("Failed to create one or more elements.")
sys.exit(1)
for element in [self.v_source0, self.a_source0, self.v_source1, self.a_source1, self.a_mixer, self.a_convert, self.a_resample, self.a_sink, self.v_compositor, self.v_convert, self.v_queue, self.v_sink]:
self.pipeline.add(element)
element.sync_state_with_parent()
# audiomixer -> audio_convert -> resample -> audio_sink
if not self.a_mixer.link(self.a_convert) or not self.a_convert.link(self.a_resample) or not self.a_resample.link(self.a_sink):
logger.error("Not all audio elements could be linked")
sys.exit(1)
# compositor -> video_convert -> queue -> video_sink
if not self.v_compositor.link(self.v_convert) or not self.v_convert.link(self.v_queue) or not self.v_queue.link(self.v_sink):
logger.error("Failed to link video convert to sink")
self.v_source_pad0 = self.v_compositor.get_request_pad("sink_%u")
self.v_source_pad1 = self.v_compositor.get_request_pad("sink_%u")
self.v_pads = [self.v_source_pad0, self.v_source_pad1]
self.v_source0.set_property("uri", "file:///home/user/video0.mp4")
self.v_source0.connect("pad-added", self._video_pad_added_handler)
self.v_source1.set_property("uri", "file:///home/user/video1.mp4")
self.v_source1.connect("pad-added", self._video_pad_added_handler)
self.a_source_pad0 = self.a_mixer.get_request_pad("sink_%u")
self.a_source_pad1 = self.a_mixer.get_request_pad("sink_%u")
self.a_pads = [self.a_source_pad0, self.a_source_pad1]
self.a_source0.set_property("uri", "file:///home/user/audio0.wav")
self.a_source0.connect("pad-added", self._audio_pad_added_handler)
self.a_source1.set_property("uri", "file:///home/user/audio1.wav")
self.a_source1.connect("pad-added", self._audio_pad_added_handler)
self.input_handler = create_input_handler(InputType.KEYBOARD, self)
self.input_handler.start()
logger.info("Im gonna set pipeline to playing now...")
ret = self.pipeline.set_state(Gst.State.PLAYING)
if ret == Gst.StateChangeReturn.FAILURE:
logger.error("Failed to set pipeline to playing state")
sys.exit(1)
logger.info("Pipeline should be playing now...")
self.loop = GLib.MainLoop()
bus = self.pipeline.get_bus()
bus.add_signal_watch()
bus.connect("message", self._on_bus_message)
self.switch_to(self.current_index)
try:
self.loop.run()
except KeyboardInterrupt:
logger.info("GLib Loop Interrupted")
finally:
self.quit()
def _on_bus_message(self, bus, msg):
if msg:
if msg.type == Gst.MessageType.ERROR:
err, debug_info = msg.parse_error()
logger.error(f"Error received from element {msg.src.get_name()}: {err.message}")
logger.error(f"Debugging information: {debug_info if debug_info else 'none'}")
self.quit()
elif msg.type == Gst.MessageType.EOS:
logger.info("End-Of-Stream reached.")
self.quit()
elif msg.type == Gst.MessageType.STATE_CHANGED:
# We are only interested in state-changed messages from the pipeline
if msg.src == self.pipeline:
old_state, new_state, pending_state = msg.parse_state_changed()
logger.info(f"Pipeline state changed from {old_state} to {new_state}")
else:
# We should not reach here
logger.error("Unexpected message received.")
# 'src' is the element that owns the given pad
# 'new_pad' is the specific pad that was added. This is usually twice per src, a pad for video and a pad for audio
def _video_pad_added_handler(self, src, new_pad):
logger.info(f"Video - Attempting to dynamically select pad: {int(src.name[-1])}")
v_sink_pad = self.v_pads[int(src.name[-1])]
logger.info(f"Video received new pad '{new_pad.name}' from '{src.name}'")
if v_sink_pad.is_linked():
logger.info("Video already linked, ignoring")
sys.exit()
new_pad_caps = new_pad.get_current_caps()
new_pad_struct = new_pad_caps.get_structure(0)
new_pad_type = new_pad_struct.get_name()
if new_pad_type.startswith("video/x-raw"):
logger.info("Video attempting to link to audio sink")
ret = new_pad.link(v_sink_pad)
if ret != Gst.PadLinkReturn.OK:
logger.error(f"Video - Type is '{new_pad_type}' but link failed")
else:
logger.info(f"Video link successful of type '{new_pad_type}'")
else:
logger.warning(f"Video pad handler received non-video '{new_pad_type}'")
def _audio_pad_added_handler(self, src, new_pad):
logger.info(f"Audio - Attempting to dynamically select pad: {int(src.name[-1])}")
a_sink_pad = self.a_pads[int(src.name[-1])]
logger.info(f"Received new pad '{new_pad.name}' from '{src.name}'")
if a_sink_pad.is_linked():
logger.info("Audio already linked, ignoring")
sys.exit()
new_pad_caps = new_pad.get_current_caps()
new_pad_struct = new_pad_caps.get_structure(0)
new_pad_type = new_pad_struct.get_name()
if new_pad_type.startswith("audio/x-raw"):
logger.info("Attempting to link to audio sink")
ret = new_pad.link(a_sink_pad)
if ret != Gst.PadLinkReturn.OK:
logger.error(f"Type is '{new_pad_type}' but link failed")
else:
logger.info(f"Audio link successful of type '{new_pad_type}'")
else:
logger.warning(f"Audio pad handler received non-audio '{new_pad_type}'")
# ------- SWITCHING --------
def switch_to(self, index):
self.current_index = index
logger.info(f"Actual switch to {self.current_index}")
self.v_pads[index].set_property("alpha", 1.0)
self.a_pads[index].set_property("volume", 1.0)
for i in range(len(self.v_pads)):
logger.info(f"i is {i}")
if i == self.current_index:
logger.info(f"Skipping {i}")
continue
self.v_pads[i].set_property("alpha", 0.0)
self.a_pads[i].set_property("volume", 0.0)
# -------- CONTROLS --------
def quit(self):
try:
self.pipeline.set_state(Gst.State.NULL)
except:
logger.info("Pipeline to NULL state threw exception")
try:
self.loop.quit()
except:
logger.info("Loop quit threw exception")
pass
try:
self.input_handler.stop()
except:
logger.info("Input handler stop threw exception")
video = VideoHandler()
