I am using gstreamer 1.22 operating a pipeline like the following:
rtspsrc -> rtph264depay -> h264parse -> kvssink
Occasionally in some environments with flaky network connectivity to the rtsp stream IP, an issue will occur and rtspsrc will emit an EOS, in response I try to shut down the pipeline via the following:
pipeline.set_state(NULL);
This will cause the pipeline to permanently deadlock. I have looked at many related filed Gstreamer bugs such as:
https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/-/issues/900
https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/-/issues/876
Which seem to be outlining the same behavior, but none of the applied fixes seem to be working in the latest version for me.
I have filed my own bug report on the issue that seems to be sitting idle.
https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/3101
Any idea what the cause of this could be, or what some things I could try would be?
Here are some thread traces for impacted pipelines:
..
#2 0x00007f7804ffa5aa in g_rec_mutex_lock (mutex=mutex@entry=0x7f736c06f378) at ../glib/gthread-posix.c:395
#3 0x00007f775fadc733 in gst_rtspsrc_change_state (element=0x7f736c06f1f0, transition=<optimized out>) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:9525
...
#9 0x00007f780557ef9e in gst_element_change_state (element=element@entry=0x7f735c003190, transition=transition@entry=GST_STATE_CHANGE_PLAYING_TO_PAUSED) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstelement.c:3093
#10 0x00007f780557f68b in gst_element_set_state_func (element=0x7f735c003190, state=GST_STATE_NULL) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstelement.c:3047
#0 0x00007f79473003ad in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f79472f9d6a in pthread_mutex_lock () from /lib64/libpthread.so.0
#2 0x00007f7804ffa5aa in g_rec_mutex_lock (mutex=mutex@entry=0x7f7424018d00) at ../glib/gthread-posix.c:395
#3 0x00007f780559b333 in gst_pad_send_event_unchecked (pad=pad@entry=0x7f7424018c90, event=<optimized out>, event@entry=0x7f74300081d0, type=type@entry=GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:5892
#4 0x00007f780559be13 in gst_pad_push_event_unchecked (pad=pad@entry=0x7f73b0044540, event=0x7f74300081d0, type=type@entry=GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:5572
#5 0x00007f780559c7f0 in push_sticky (user_data=<synthetic pointer>, ev=<synthetic pointer>, pad=0x7f73b0044540) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4057
#6 events_foreach (user_data=<synthetic pointer>, func=0x7f780559b780 <push_sticky>, pad=0x7f73b0044540) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:613
#7 check_sticky (pad=0x7f73b0044540, event=0x7f74300081d0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4116
#8 0x00007f78055a6108 in gst_pad_push_event (pad=0x7f73b0044540, event=0x7f74300081d0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:5705
#9 0x00007f775fadcc4e in gst_rtspsrc_stream_push_event (src=0x7f736c06f1f0, event=0x7f7430008160, stream=0x7f73b0016110) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:5217
#10 on_timeout_common (session=<optimized out>, source=<optimized out>, stream=<optimized out>) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:3726
#11 0x00007f78052c7a97 in g_closure_invoke (closure=0x7f73b0043fa0, return_value=0x0, n_param_values=2, param_values=0x7f71802909d0, invocation_hint=0x7f7180290970) at ../gobject/gclosure.c:810
#12 0x00007f78052d9930 in signal_emit_unlocked_R (node=node@entry=0x7f77c815c700, detail=detail@entry=0, instance=instance@entry=0x7f7424010400, emission_return=emission_return@entry=0x0, instance_and_params=instance_and_params@entry=0x7f71802909d0) at ../gobject/gsignal.c:3741
#13 0x00007f78052e3f31 in g_signal_emit_valist (instance=<optimized out>, signal_id=<optimized out>, detail=<optimized out>, var_args=var_args@entry=0x7f7180290b78) at ../gobject/gsignal.c:3497
#14 0x00007f78052e4982 in g_signal_emit (instance=instance@entry=0x7f7424010400, signal_id=<optimized out>, detail=detail@entry=0) at ../gobject/gsignal.c:3553
#15 0x00007f7747c7ec24 in on_timeout (source=0x7f73f8003110, sess=0x7f7424010400) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c:1126
#16 session_cleanup (key=<optimized out>, source=0x7f73f8003110, data=0x7f7180290d40) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c:4243
#17 0x00007f7804f892af in g_hash_table_foreach (hash_table=hash_table@entry=0x7f7430007520, func=func@entry=0x7f7747c7e3b0 <session_cleanup>, user_data=user_data@entry=0x7f7180290d40) at ../glib/ghash.c:2065
#18 0x00007f7747c879bb in rtp_session_on_timeout (sess=sess@entry=0x7f7424010400, current_time=current_time@entry=34977333832252, ntpnstime=<optimized out>, running_time=<optimized out>) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c:4696
#0 0x00007f2ea53832a9 in syscall () from /lib64/libc.so.6
#1 0x00007f2cef06f26f in g_cond_wait (cond=cond@entry=0x7f29c41d2030, mutex=mutex@entry=0x7f29c41d2020) at ../glib/gthread-posix.c:1574
#2 0x00007f2c9822d468 in gst_rtp_jitter_buffer_chain (pad=<optimized out>, parent=0x7f29c41d2380, buffer=<optimized out>) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c:3427
#3 0x00007f2cef60e618 in gst_pad_chain_data_unchecked (pad=pad@entry=0x7f29c420dc90, type=type@entry=4112, data=data@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4463
#4 0x00007f2cef612184 in gst_pad_push_data (pad=pad@entry=0x7f29c420d5a0, type=type@entry=4112, data=data@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4739
#5 0x00007f2cef618f0e in gst_pad_push (pad=pad@entry=0x7f29c420d5a0, buffer=buffer@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4858
#6 0x00007f2c9823eb4d in gst_rtp_ssrc_demux_chain (pad=<optimized out>, parent=0x7f2a30019b80, buf=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.c:680
#7 0x00007f2cef60e618 in gst_pad_chain_data_unchecked (pad=pad@entry=0x7f29d010dd10, type=type@entry=4112, data=data@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4463
#8 0x00007f2cef612184 in gst_pad_push_data (pad=pad@entry=0x7f29d010dac0, type=type@entry=4112, data=data@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4739
#9 0x00007f2cef618f0e in gst_pad_push (pad=0x7f29d010dac0, buffer=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4858
#10 0x00007f2cef60e618 in gst_pad_chain_data_unchecked (pad=pad@entry=0x7f29c420c0d0, type=type@entry=4112, data=data@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4463
#11 0x00007f2cef612184 in gst_pad_push_data (pad=pad@entry=0x7f29c420c570, type=type@entry=4112, data=data@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4739
#12 0x00007f2cef618f0e in gst_pad_push (pad=pad@entry=0x7f29c420c570, buffer=buffer@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4858
#13 0x00007f2c9825f451 in gst_rtp_session_process_rtp (sess=<optimized out>, src=<optimized out>, buffer=0x7f2bf88996c0, user_data=0x7f2b94021760) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c:1420
#14 0x00007f2c98244722 in source_push_rtp (source=0x7f2a0413f6f0, data=0x7f2bf88996c0, session=0x7f2bf00483e0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c:1566
#15 0x00007f2c98256004 in push_packet (buffer=<optimized out>, src=0x7f2a0413f6f0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c:935
#16 rtp_source_process_rtp (src=src@entry=0x7f2a0413f6f0, pinfo=pinfo@entry=0x7f2a08bfb6f0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c:1344
#17 0x00007f2c982518f6 in rtp_session_process_rtp (sess=0x7f2bf00483e0, buffer=buffer@entry=0x7f2bf88996c0, current_time=<optimized out>, running_time=<optimized out>, ntpnstime=<optimized out>)
at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c:2357
#18 0x00007f2c98263995 in gst_rtp_session_chain_recv_rtp (pad=<optimized out>, parent=0x7f2b94021760, buffer=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c:2013
#19 0x00007f2cef60e618 in gst_pad_chain_data_unchecked (pad=pad@entry=0x7f29c420c320, type=type@entry=4112, data=data@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4463
#20 0x00007f2cef612184 in gst_pad_push_data (pad=pad@entry=0x7f29c80082f0, type=type@entry=4112, data=data@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4739
#21 0x00007f2cef618f0e in gst_pad_push (pad=pad@entry=0x7f29c80082f0, buffer=buffer@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4858
#22 0x00007f2cef5fae02 in gst_proxy_pad_chain_default (pad=<optimized out>, parent=<optimized out>, buffer=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstghostpad.c:127
#23 0x00007f2cef60e618 in gst_pad_chain_data_unchecked (pad=pad@entry=0x7f29c80062c0, type=type@entry=4112, data=data@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4463
#24 0x00007f2cef612184 in gst_pad_push_data (pad=pad@entry=0x7f29c420cc60, type=type@entry=4112, data=data@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4739
#25 0x00007f2cef618f0e in gst_pad_push (pad=pad@entry=0x7f29c420cc60, buffer=buffer@entry=0x7f2bf88996c0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4858
#26 0x00007f2cedb77c65 in gst_rtspsrc_handle_data (src=src@entry=0x7f29bc121650, message=message@entry=0x7f2a08bfbd90) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:5789
#27 0x00007f2cedb8b41c in gst_rtspsrc_loop_interleaved (src=0x7f29bc121650) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:5869
#28 gst_rtspsrc_loop (src=0x7f29bc121650) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:6366
#29 gst_rtspsrc_thread (src=0x7f29bc121650) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:9378
#30 0x00007f2cef64af96 in gst_task_func (task=0x7f2bf805bef0) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gsttask.c:384
#31 0x00007f2cef04322d in g_thread_pool_thread_proxy (data=<optimized out>) at ../glib/gthreadpool.c:354
#32 0x00007f2cef042b1d in g_thread_proxy (data=0x7f2a1001a580) at ../glib/gthread.c:826
#33 0x00007f2ea585144b in start_thread () from /lib64/libpthread.so.0
#34 0x00007f2ea538852f in clone () from /lib64/libc.so.6