I was noticing that there is artifacting due to packet loss in webrtcbin. So I was adviced to set the do-nack to true and fec-type to UlpRed.
I have a pipeline like so
let pipeline = gstreamer::parse_launch(
"webrtcbin name=webrtcbin latency=100 ! decodebin ! queue ! videoscale ! waylandsink name=videosink"
).unwrap();
Referring to webrtcsrc implementation, I have tweaked it like so
let mut opus_id = None;
let mut video_id = None;
for (i, media) in sdp_message.medias().enumerate() {
for fmt in media.formats() {
if fmt == "webrtc-datachannel" {
continue;
}
let pt = match fmt.parse::<u8>() {
Ok(pt) => pt,
Err(_) => continue
};
let caps = match media.caps_from_media(pt as i32) {
Some(caps) if caps.size() > 0 => caps,
_ => continue
};
let s = caps.structure(0).unwrap();
let encoding_name = match s.get::<&str>("encoding-name") {
Ok(encoding_name) => encoding_name,
Err(_) => continue
};
let twcc_id = media.attributes().find_map(|attr| {
let key = attr.key();
let value = attr.value();
if key != "extmap" || !value.map_or(false, |value| value.ends_with(TWCC_URI)) {
return None;
}
let value = value.unwrap();
let id = value.strip_suffix(TWCC_URI).and_then(|twcc_id| twcc_id.trim().parse::<u8>().ok());
id
if encoding_name == "VP8" && video_id.is_none() {
video_id = Some((pt, twcc_id));
}
else if encoding_name == "OPUS" && opus_id.is_none() {
opus_id = Some((pt, twcc_id));
let mut filtered_s = gstreamer::Structure::new_empty("application/x-rtp");
filtered_s.extend(s.iter().filter_map(|(key, value)| {
if !key.starts_with("rtcp-") || key.starts_with("extmap-") {
Some((key, value.to_owned()))
}
else {
None
}
}));
if !caps.is_empty() {
let transceiver = webrtcbin.emit_by_name::<Option<gstreamer_webrtc::WebRTCRTPTransceiver>>("get-transceiver", &[&(i as i32)])
.unwrap_or_else(|| {
webrtcbin.emit_by_name::<gstreamer_webrtc::WebRTCRTPTransceiver>(
"add-transceiver",
&[&gstreamer_webrtc::WebRTCRTPTransceiverDirection::Recvonly,
});
println!("{:?} {:?}", transceiver.has_property("do_nack", None), transceiver.has_property("do-nack", None));
transceiver.set_property("do-nack", true);
transceiver.set_property("fec-type", gstreamer_webrtc::WebRTCFECType::UlpRed);
transceiver.set_property("codec-preferences", caps);
}
}
}
// Set remote description
Answer being sent
v=0
o=- 8174134731741160235 3 IN IP4 127.0.0.1
s=-
t=0 0
a=group:BUNDLE 0 1
a=extmap-allow-mixed
a=msid-semantic: WMS 39bc895f-e4aa-4a20-be3e-5ee684ee59bc
m=application 52335 UDP/DTLS/SCTP webrtc-datachannel
c=IN IP4 64.227.163.77
a=candidate:3455220146 1 udp 2113937151 fedc4764-0273-4ff5-be3f-76d08e9907bb.local 50196 typ host generation 0 network-cost 999
a=candidate:3632902196 1 udp 2113939711 e61ad7e9-8778-44a7-bea3-3fcb87de4a5b.local 54444 typ host generation 0 network-cost 999
a=candidate:1400217228 1 udp 1677729535 49.207.246.19 16022 typ srflx raddr 0.0.0.0 rport 0 generation 0 network-cost 999
a=candidate:4233327282 1 udp 33562623 64.227.163.77 52335 typ relay raddr 49.207.246.19 rport 16022 generation 0 network-cost 999
a=ice-ufrag:45ma
a=ice-pwd:+vlNPir6G62secizkzRrBNEc
a=ice-options:trickle
a=fingerprint:sha-256 B2:A7:7D:6D:19:2F:87:7D:18:A7:E7:F2:A7:F3:1F:B8:C3:F9:5F:FF:0E:80:99:94:DA:5C:9F:E0:45:50:DB:12
a=setup:actpass
a=mid:0
a=sctp-port:5000
a=max-message-size:262144
m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 116 117 118
c=IN IP4 0.0.0.0
a=rtcp:9 IN IP4 0.0.0.0
a=ice-ufrag:45ma
a=ice-pwd:+vlNPir6G62secizkzRrBNEc
a=ice-options:trickle
a=fingerprint:sha-256 B2:A7:7D:6D:19:2F:87:7D:18:A7:E7:F2:A7:F3:1F:B8:C3:F9:5F:FF:0E:80:99:94:DA:5C:9F:E0:45:50:DB:12
a=setup:actpass
a=mid:1
a=extmap:1 urn:ietf:params:rtp-hdrext:toffset
a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
a=extmap:3 urn:3gpp:video-orientation
a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay
a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type
a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing
a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space
a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid
a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id
a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
a=sendonly
a=msid:39bc895f-e4aa-4a20-be3e-5ee684ee59bc c14c0477-c1b0-4a79-b7bd-f10e4bc7cab7
a=rtcp-mux
a=rtcp-rsize
a=rtpmap:96 VP8/90000
a=rtcp-fb:96 goog-remb
a=rtcp-fb:96 transport-cc
a=rtcp-fb:96 ccm fir
a=rtcp-fb:96 nack
a=rtcp-fb:96 nack pli
a=rtpmap:97 rtx/90000
a=fmtp:97 apt=96
a=rtpmap:102 H264/90000
a=rtcp-fb:102 goog-remb
a=rtcp-fb:102 transport-cc
a=rtcp-fb:102 ccm fir
a=rtcp-fb:102 nack
a=rtcp-fb:102 nack pli
a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f
a=rtpmap:103 rtx/90000
a=fmtp:103 apt=102
a=rtpmap:104 H264/90000
a=rtcp-fb:104 goog-remb
a=rtcp-fb:104 transport-cc
a=rtcp-fb:104 ccm fir
a=rtcp-fb:104 nack
a=rtcp-fb:104 nack pli
a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f
a=rtpmap:105 rtx/90000
a=fmtp:105 apt=104
a=rtpmap:106 H264/90000
a=rtcp-fb:106 goog-remb
a=rtcp-fb:106 transport-cc
a=rtcp-fb:106 ccm fir
a=rtcp-fb:106 nack
a=rtcp-fb:106 nack pli
a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f
a=rtpmap:107 rtx/90000
a=fmtp:107 apt=106
a=rtpmap:108 H264/90000
a=rtcp-fb:108 goog-remb
a=rtcp-fb:108 transport-cc
a=rtcp-fb:108 ccm fir
a=rtcp-fb:108 nack
a=rtcp-fb:108 nack pli
a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f
a=rtpmap:109 rtx/90000
a=fmtp:109 apt=108
a=rtpmap:127 H264/90000
a=rtcp-fb:127 goog-remb
a=rtcp-fb:127 transport-cc
a=rtcp-fb:127 ccm fir
a=rtcp-fb:127 nack
a=rtcp-fb:127 nack pli
a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f
a=rtpmap:125 rtx/90000
a=fmtp:125 apt=127
a=rtpmap:39 H264/90000
a=rtcp-fb:39 goog-remb
a=rtcp-fb:39 transport-cc
a=rtcp-fb:39 ccm fir
a=rtcp-fb:39 nack
a=rtcp-fb:39 nack pli
a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f
a=rtpmap:40 rtx/90000
a=fmtp:40 apt=39
a=rtpmap:45 AV1/90000
a=rtcp-fb:45 goog-remb
a=rtcp-fb:45 transport-cc
a=rtcp-fb:45 ccm fir
a=rtcp-fb:45 nack
a=rtcp-fb:45 nack pli
a=fmtp:45 level-idx=5;profile=0;tier=0
a=rtpmap:46 rtx/90000
a=fmtp:46 apt=45
a=rtpmap:98 VP9/90000
a=rtcp-fb:98 goog-remb
a=rtcp-fb:98 transport-cc
a=rtcp-fb:98 ccm fir
a=rtcp-fb:98 nack
a=rtcp-fb:98 nack pli
a=fmtp:98 profile-id=0
a=rtpmap:99 rtx/90000
a=fmtp:99 apt=98
a=rtpmap:100 VP9/90000
a=rtcp-fb:100 goog-remb
a=rtcp-fb:100 transport-cc
a=rtcp-fb:100 ccm fir
a=rtcp-fb:100 nack
a=rtcp-fb:100 nack pli
a=fmtp:100 profile-id=2
a=rtpmap:101 rtx/90000
a=fmtp:101 apt=100
a=rtpmap:112 H264/90000
a=rtcp-fb:112 goog-remb
a=rtcp-fb:112 transport-cc
a=rtcp-fb:112 ccm fir
a=rtcp-fb:112 nack
a=rtcp-fb:112 nack pli
a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f
a=rtpmap:113 rtx/90000
a=fmtp:113 apt=112
a=rtpmap:116 red/90000
a=rtpmap:117 rtx/90000
a=fmtp:117 apt=116
a=rtpmap:118 ulpfec/90000
a=ssrc-group:FID 4056307815 3229411600
a=ssrc:4056307815 cname:ZCN15yuiKccGbgey
a=ssrc:4056307815 msid:39bc895f-e4aa-4a20-be3e-5ee684ee59bc c14c0477-c1b0-4a79-b7bd-f10e4bc7cab7
a=ssrc:3229411600 cname:ZCN15yuiKccGbgey
a=ssrc:3229411600 msid:39bc895f-e4aa-4a20-be3e-5ee684ee59bc c14c0477-c1b0-4a79-b7bd-f10e4bc7cab7
So from the answer above it’s clear that rtx has been agreed upon.
But then, I see 0 nacks in the stats despite there being packet loss.
For comparison, here are the screenshorts of the chrome://webrtc-internals in the case of the browser where nacks are sent and in the case of GStreamer where it is not.
GStreamer
Browser
