Trying to build a webrtc on local network using webrtcbin

Hello,
I am trying to build a webRTC app on my local network. I am using webrtcbin and GStreamer in v1.16. My hardware is a Jetson.

This the GStreamer app :

// Global variables
static GMainLoop *loop;
static GstElement *pipeline, *webrtcbin;
static SoupWebsocketConnection *ws_conn = NULL;

// Function declarations
static void send_sdp_offer(GstWebRTCSessionDescription *offer);
static void on_offer_created(GstPromise *promise, gpointer user_data);
static void on_negotiation_needed(GstElement *element, gpointer user_data);
static void on_ice_candidate(GstElement *webrtc, guint mline_index,
                             gchar *candidate, gpointer user_data);
static void soup_websocket_message(SoupWebsocketConnection *conn,
                                   SoupWebsocketDataType type, GBytes *message,
                                   gpointer user_data);
static void on_server_connected(GObject *source_object, GAsyncResult *res,
                                gpointer user_data);
static void on_bus_message(GstBus *bus, GstMessage *message, gpointer data);
static gchar *get_string_from_json_object(JsonObject *object);
static void handle_sdp_answer(const gchar *sdp_json_str);
static void handle_ice_candidate(const gchar *ice_json_str);

// Entry point
int main(int argc, char *argv[]) {
  GstElement *v4l2src, *videoconvert, *queue, *vp8enc, *rtpvp8pay;
  GstBus *bus;
  SoupSession *session;
  SoupMessage *message;
  const gchar *websocket_url = "ws://localhost:8443"; // Use ws:// for non-SSL

  // Initialize GStreamer
  gst_init(&argc, &argv);

  // Create a GMainLoop
  loop = g_main_loop_new(NULL, FALSE);

  // Create the GStreamer pipeline
  pipeline = gst_pipeline_new("webrtc-pipeline");
  if (!pipeline) {
    g_printerr("Failed to create pipeline.\n");
    return -1;
  }

  // Create GStreamer elements
  v4l2src = gst_element_factory_make("v4l2src", "video-source");
  if (!v4l2src) {
    g_printerr("Failed to create v4l2src element.\n");
    return -1;
  }
  g_object_set(v4l2src, "device", "/dev/video0", NULL);

  videoconvert = gst_element_factory_make("videoconvert", "videoconvert");
  if (!videoconvert) {
    g_printerr("Failed to create videoconvert element.\n");
    return -1;
  }

  queue = gst_element_factory_make("queue", "queue");
  if (!queue) {
    g_printerr("Failed to create queue element.\n");
    return -1;
  }

  vp8enc = gst_element_factory_make("vp8enc", "vp8enc");
  if (!vp8enc) {
    g_printerr("Failed to create vp8enc element.\n");
    return -1;
  }
  // Set properties for the VP8 encoder
  g_object_set(vp8enc, "deadline", 1, NULL);

  rtpvp8pay = gst_element_factory_make("rtpvp8pay", "rtpvp8pay");
  if (!rtpvp8pay) {
    g_printerr("Failed to create rtpvp8pay element.\n");
    return -1;
  }
  // Set the payload type for RTP
  g_object_set(rtpvp8pay, "pt", 96, NULL);

  webrtcbin = gst_element_factory_make("webrtcbin", "webrtcbin");
  if (!webrtcbin) {
    g_printerr("Failed to create webrtcbin element.\n");
    return -1;
  }
  // Optional: Set the STUN server
  // g_object_set(webrtcbin, "stun-server", "stun://stun.l.google.com:19302",
  // NULL);

  // Add all elements to the pipeline
  gst_bin_add_many(GST_BIN(pipeline), v4l2src, videoconvert, queue, vp8enc,
                   rtpvp8pay, webrtcbin, NULL);

  // Link the elements together
  if (!gst_element_link_many(v4l2src, videoconvert, queue, vp8enc, rtpvp8pay,
                             NULL)) {
    g_printerr("Failed to link video elements.\n");
    gst_object_unref(pipeline);
    return -1;
  }

  if (!gst_element_link_pads(rtpvp8pay, "src", webrtcbin, "sink_%u")) {
    g_printerr("Failed to link rtpvp8pay to webrtcbin.\n");
    gst_object_unref(pipeline);
    return -1;
  }

  // Get the bus from the pipeline and add a signal watch
  bus = gst_element_get_bus(pipeline);
  gst_bus_add_signal_watch(bus);
  // Connect the bus message handler
  g_signal_connect(bus, "message", G_CALLBACK(on_bus_message), NULL);
  gst_object_unref(bus);

  // Create a SoupSession for WebSocket communication
  session = soup_session_new();

  // Create a SoupMessage for the WebSocket connection
  message = soup_message_new("GET", websocket_url);

  // Initiate the WebSocket connection asynchronously
  soup_session_websocket_connect_async(
      session,             // SoupSession *
      message,             // SoupMessage *
      NULL,                // const char *origin
      NULL,                // const char **protocols
      NULL,                // GCancellable *
      on_server_connected, // GAsyncReadyCallback
      NULL                 // gpointer user_data
  );

  // Run the main loop
  g_main_loop_run(loop);

  // Clean up after exiting the main loop
  gst_element_set_state(pipeline, GST_STATE_NULL);
  gst_object_unref(pipeline);
  g_main_loop_unref(loop);

  return 0;
}

// Function to handle messages from the GStreamer bus
static void on_bus_message(GstBus *bus, GstMessage *message, gpointer data) {
  // Cast unused parameters to void
  (void)bus;
  (void)data;

  switch (GST_MESSAGE_TYPE(message)) {
  case GST_MESSAGE_ERROR: {
    GError *err = NULL;
    gchar *debug_info = NULL;
    // Parse the error message
    gst_message_parse_error(message, &err, &debug_info);
    g_printerr("Error received from element %s: %s\n",
               GST_OBJECT_NAME(message->src), err->message);
    g_printerr("Debugging information: %s\n", debug_info ? debug_info : "none");
    g_clear_error(&err);
    g_free(debug_info);
    // Exit the main loop on error
    g_main_loop_quit(loop);
    break;
  }
  case GST_MESSAGE_WARNING: {
    GError *err = NULL;
    gchar *debug_info = NULL;
    // Parse the warning message
    gst_message_parse_warning(message, &err, &debug_info);
    g_printerr("Warning received from element %s: %s\n",
               GST_OBJECT_NAME(message->src), err->message);
    g_printerr("Debugging information: %s\n", debug_info ? debug_info : "none");
    g_clear_error(&err);
    g_free(debug_info);
    break;
  }
  case GST_MESSAGE_EOS:
    // End-of-stream message
    g_print("End-Of-Stream reached.\n");
    g_main_loop_quit(loop);
    break;
  default:
    // Unhandled message type
    break;
  }
}

// Callback function when the WebSocket connection is established
static void on_server_connected(GObject *source_object, GAsyncResult *res,
                                gpointer user_data) {
  // Cast unused parameter to void
  (void)user_data;

  SoupSession *session = SOUP_SESSION(source_object);
  GError *error = NULL;

  // Finish the WebSocket connection and get the connection object
  ws_conn = soup_session_websocket_connect_finish(session, res, &error);

  if (error) {
    // Handle connection errors
    g_printerr("Failed to connect to WebSocket server: %s\n", error->message);
    g_error_free(error);
    g_main_loop_quit(loop);
    return;
  }

  // Send HELLO message to register with the signaling server
  gchar *peer_id = "drone"; // UID for this GStreamer client
  gchar *hello_message = g_strdup_printf("HELLO %s", peer_id);
  soup_websocket_connection_send_text(ws_conn, hello_message);
  g_print("Sent HELLO: %s\n", hello_message);
  g_free(hello_message);

  // Connect signal handler for WebSocket messages
  g_signal_connect(ws_conn, "message", G_CALLBACK(soup_websocket_message),
                   NULL);

  // NOTE: Do NOT connect webrtcbin signals or set the pipeline to PLAYING here.
}

// Function to handle incoming WebSocket messages
static void soup_websocket_message(SoupWebsocketConnection *conn,
                                   SoupWebsocketDataType type, GBytes *message,
                                   gpointer user_data) {
  // Cast unused parameter to void
  (void)conn;
  (void)user_data;

  gsize size;
  const guint8 *data;
  gchar *data_str;

  if (type == SOUP_WEBSOCKET_DATA_BINARY) {
    // Ignore binary messages
    g_printerr("Received unknown binary message, ignoring\n");
    return;
  }

  // Get data and size from GBytes
  data = g_bytes_get_data(message, &size);
  // Create a null-terminated string
  data_str = g_strndup((const gchar *)data, size);

  // Print the raw received message for debugging
  g_print("Received message: %s\n", data_str);

  // Handle text messages from the signaling server
  if (g_str_has_prefix(data_str, "HELLO")) {
    // Server acknowledged HELLO message
    g_print("Received HELLO from server\n");
    // No action needed upon receiving HELLO
  } else if (g_str_has_prefix(data_str, "SESSION_REQUEST ")) {
    // Handle incoming session request from the signaling server
    const gchar *caller_id = data_str + strlen("SESSION_REQUEST ");
    g_print("Received SESSION_REQUEST from %s\n", caller_id);

    // Respond with SESSION_OK to confirm the session
    gchar *session_ok_message = g_strdup("SESSION_OK");
    soup_websocket_connection_send_text(ws_conn, session_ok_message);
    g_print("Sent SESSION_OK to server\n");
    g_free(session_ok_message);

    // Now that the session is established, connect the WebRTC signals
    g_signal_connect(webrtcbin, "on-negotiation-needed",
                     G_CALLBACK(on_negotiation_needed), NULL);
    g_signal_connect(webrtcbin, "on-ice-candidate",
                     G_CALLBACK(on_ice_candidate), NULL);

    // Set the pipeline to PLAYING state
    gst_element_set_state(pipeline, GST_STATE_PLAYING);
    g_print("Pipeline set to PLAYING state\n");
  } else if (g_str_has_prefix(data_str, "SESSION_OK")) {
    g_print("Received SESSION_OK from server\n");
    // Session is established, proceed with WebRTC negotiation

    // Now that the session is established, connect the WebRTC signals
    g_signal_connect(webrtcbin, "on-negotiation-needed",
                     G_CALLBACK(on_negotiation_needed), NULL);
    g_signal_connect(webrtcbin, "on-ice-candidate",
                     G_CALLBACK(on_ice_candidate), NULL);

    // Set the pipeline to PLAYING state
    gst_element_set_state(pipeline, GST_STATE_PLAYING);
    g_print("Pipeline set to PLAYING state\n");
  } else if (g_str_has_prefix(data_str, "SDP_ANSWER ")) {
    // Handle SDP answer from the browser
    const gchar *sdp_json_str = data_str + strlen("SDP_ANSWER ");
    g_print("Handling SDP_ANSWER: %s\n", sdp_json_str);
    handle_sdp_answer(sdp_json_str);
  } else if (g_str_has_prefix(data_str, "ICE_CANDIDATE ")) {
    // Handle ICE candidate from the browser
    const gchar *ice_json_str = data_str + strlen("ICE_CANDIDATE ");
    g_print("Handling ICE_CANDIDATE: %s\n", ice_json_str);
    handle_ice_candidate(ice_json_str);
  } else {
    g_printerr("Unknown message: %s\n", data_str);
    // Optionally, ignore or handle other message types
  }

  // Free the allocated string
  g_free(data_str);
}

// Function to handle the negotiation needed signal from webrtcbin
static void on_negotiation_needed(GstElement *element, gpointer user_data) {
  // Cast unused parameters to void
  (void)element;
  (void)user_data;

  GstPromise *promise;

  // Create a promise to receive the SDP offer
  promise = gst_promise_new_with_change_func(on_offer_created, NULL, NULL);
  // Ask webrtcbin to create an offer
  g_signal_emit_by_name(webrtcbin, "create-offer", NULL, promise);
  g_print("Created SDP offer and waiting for it to be ready\n");
}

// Callback function when the SDP offer is created
static void on_offer_created(GstPromise *promise, gpointer user_data) {
  // Cast unused parameter to void
  (void)user_data;

  GstWebRTCSessionDescription *offer = NULL;
  const GstStructure *reply;

  // Wait for the promise to be fulfilled
  g_assert_cmphex(gst_promise_wait(promise), ==, GST_PROMISE_RESULT_REPLIED);
  // Get the reply from the promise
  reply = gst_promise_get_reply(promise);
  gst_structure_get(reply, "offer", GST_TYPE_WEBRTC_SESSION_DESCRIPTION, &offer,
                    NULL);
  gst_promise_unref(promise);

  // Set the local description on webrtcbin
  g_signal_emit_by_name(webrtcbin, "set-local-description", offer, NULL);
  g_print("Set local description with SDP offer\n");

  // Send the SDP offer to the browser via WebSocket
  send_sdp_offer(offer);
  gst_webrtc_session_description_free(offer);
}

// Function to send the SDP offer to the browser
static void send_sdp_offer(GstWebRTCSessionDescription *offer) {
  gchar *sdp_str = gst_sdp_message_as_text(offer->sdp);
  JsonObject *sdp_json = json_object_new();
  // Create a JSON object for the SDP
  json_object_set_string_member(sdp_json, "type", "offer");
  json_object_set_string_member(sdp_json, "sdp", sdp_str);

  gchar *sdp_json_str = get_string_from_json_object(sdp_json);

  gchar *message = g_strdup_printf("SDP_OFFER %s", sdp_json_str);
  soup_websocket_connection_send_text(ws_conn, message);
  g_print("Sent SDP_OFFER to browser: %s\n", message);

  // Clean up
  json_object_unref(sdp_json);
  g_free(sdp_json_str);
  g_free(sdp_str);
  g_free(message);
}

// Function to handle the ICE candidate received from the browser
static void handle_ice_candidate(const gchar *ice_json_str) {
  JsonParser *parser = json_parser_new();
  GError *error = NULL;

  // Print the received ICE candidate JSON for debugging
  g_print("Received ICE candidate JSON: %s\n", ice_json_str);

  // Parse the ICE candidate JSON
  if (!json_parser_load_from_data(parser, ice_json_str, -1, &error)) {
    g_printerr("Failed to parse ICE candidate JSON: %s\n", error->message);
    g_error_free(error);
    g_object_unref(parser);
    return;
  }

  JsonNode *root = json_parser_get_root(parser);
  JsonObject *ice_json = json_node_get_object(root);

  guint mline_index = json_object_get_int_member(ice_json, "sdpMLineIndex");
  const gchar *candidate = json_object_get_string_member(ice_json, "candidate");

  // Validate the candidate string
  if (candidate == NULL || strlen(candidate) == 0) {
    g_printerr("Received empty ICE candidate, ignoring\n");
    g_object_unref(parser);
    return;
  }

  // Add the ICE candidate to webrtcbin
  g_signal_emit_by_name(webrtcbin, "add-ice-candidate", mline_index, candidate);
  g_print("Added ICE_CANDIDATE from browser: %s\n", candidate);

  g_object_unref(parser);
}

// Function to handle the SDP answer received from the browser
static void handle_sdp_answer(const gchar *sdp_json_str) {
  JsonParser *parser = json_parser_new();
  GError *error = NULL;

  // Print the received SDP answer JSON for debugging
  g_print("Received SDP answer JSON: %s\n", sdp_json_str);

  // Parse the SDP answer JSON
  if (!json_parser_load_from_data(parser, sdp_json_str, -1, &error)) {
    g_printerr("Failed to parse SDP answer JSON: %s\n", error->message);
    g_error_free(error);
    g_object_unref(parser);
    return;
  }

  JsonNode *root = json_parser_get_root(parser);
  JsonObject *sdp_json = json_node_get_object(root);

  const gchar *type_str = json_object_get_string_member(sdp_json, "type");
  const gchar *sdp_str = json_object_get_string_member(sdp_json, "sdp");

  // Print the SDP type and SDP string for debugging
  g_print("SDP Type: %s\n", type_str);
  g_print("SDP: %s\n", sdp_str);

  if (g_strcmp0(type_str, "answer") != 0) {
    g_printerr("Received SDP of type '%s', expected 'answer'\n", type_str);
    g_object_unref(parser);
    return;
  }

  GstSDPMessage *sdp;
  GstWebRTCSessionDescription *answer;
  GstPromise *promise = gst_promise_new();

  // Allocate a new SDP message
  if (gst_sdp_message_new(&sdp) != GST_SDP_OK) {
    g_printerr("Failed to allocate SDP message\n");
    g_object_unref(parser);
    return;
  }
  // Parse the SDP message from the string
  if (gst_sdp_message_parse_buffer((guint8 *)sdp_str, strlen(sdp_str), sdp) !=
      GST_SDP_OK) {
    g_printerr("Failed to parse SDP message\n");
    gst_sdp_message_free(sdp);
    g_object_unref(parser);
    return;
  }

  // Create a WebRTC session description for the answer
  answer = gst_webrtc_session_description_new(GST_WEBRTC_SDP_TYPE_ANSWER, sdp);
  // Set the remote description on webrtcbin
  g_signal_emit_by_name(webrtcbin, "set-remote-description", answer, promise);
  g_print("Set remote description with SDP answer\n");
  gst_promise_interrupt(promise);
  gst_promise_unref(promise);
  gst_webrtc_session_description_free(answer);
  g_object_unref(parser);

  g_print("Processed SDP_ANSWER from browser\n");
}

// Function to serialize a JsonObject to a gchar * string
static gchar *get_string_from_json_object(JsonObject *object) {
  JsonNode *root = json_node_new(JSON_NODE_OBJECT);
  json_node_set_object(root, object);
  JsonGenerator *gen = json_generator_new();
  json_generator_set_root(gen, root);
  gchar *data = json_generator_to_data(gen, NULL);
  // Clean up
  g_object_unref(gen);
  json_node_free(root);
  return data;
}

// Function to handle the ICE candidate signal from webrtcbin
static void on_ice_candidate(GstElement *webrtc, guint mline_index,
                             gchar *candidate, gpointer user_data) {
  // Cast unused parameters to void
  (void)webrtc;
  (void)user_data;

  // Create a JSON object for the ICE candidate
  JsonObject *ice_json = json_object_new();
  json_object_set_string_member(ice_json, "candidate", candidate);
  json_object_set_int_member(ice_json, "sdpMLineIndex", mline_index);

  // Serialize the JSON object to a string
  gchar *ice_json_str = get_string_from_json_object(ice_json);

  // Prepare the message according to the protocol
  gchar *message = g_strdup_printf("ICE_CANDIDATE %s", ice_json_str);

  // Send the ICE candidate to the browser via WebSocket
  soup_websocket_connection_send_text(ws_conn, message);
  g_print("Sent ICE_CANDIDATE to browser: %s\n", message);

  // Clean up
  json_object_unref(ice_json);
  g_free(ice_json_str);
  g_free(message);
}

I am using the signaling server present in the GStreamer repos without ssl.

And finally this is the frontend:

// webrtc_client.js
const uid = "browser";
const calleeId = "drone";
const signalingServerUrl = "ws://localhost:8443";

let signalingSocket;
let peerConnection;
let localStream;
let remoteIceCandidatesQueue = [];
let remoteDescriptionSet = false;

// DOM Elements
const localVideo = document.getElementById('localVideo');
const remoteVideo = document.getElementById('remoteVideo');
const startButton = document.getElementById('startButton');
const hangupButton = document.getElementById('hangupButton');

const configuration = {
  iceServers: [],
  iceTransportPolicy: 'all',
  iceCandidatePoolSize: 0
};

/**
 * Step 1: Establish WebSocket Connection and Handshake
 */
function connectToSignalingServer() {
  // Create a new WebSocket connection to the signaling server
  signalingSocket = new WebSocket(signalingServerUrl);

  // Event handler for when the WebSocket connection is open
  signalingSocket.onopen = function () {
    console.log("Connected to the signaling server");
    // Send a handshake message to the signaling server
    signalingSocket.send(`HELLO ${uid}`);
  };

  // Event handler for receiving messages from the signaling server
  signalingSocket.onmessage = async (message) => {
    const data = message.data;
    console.log("Received message:", data);

    // Handle different types of messages based on the protocol
    if (data === 'HELLO') {
      // Step 1: Handshake completed
      console.log('Received HELLO from server');
      // Enable the start button to allow initiating a session
      startButton.disabled = false;
    } else if (data.startsWith('SESSION_REQUEST')) {
      // Handle incoming session requests
      const callerId = data.split(' ')[1];
      console.log(`Received session request from ${callerId}`);
    } else if (data === 'SESSION_OK') {
      // Handle session confirmation
      console.log('Received SESSION_OK from server');
    } else if (data.startsWith('SDP_OFFER')) {
      // Step 3: Handle the received SDP offer
      const sdpOffer = data.substring('SDP_OFFER '.length);
      await handleSdpOffer(sdpOffer);
    } else if (data.startsWith('SDP_ANSWER')) {
      // Handle SDP answer if client A was the offerer
      const sdpAnswer = data.substring('SDP_ANSWER '.length);
      await handleSdpAnswer(sdpAnswer);
    } else if (data.startsWith('ICE_CANDIDATE')) {
      // Handle ICE candidate messages
      const iceCandidate = data.substring('ICE_CANDIDATE '.length);
      await handleRemoteIceCandidate(iceCandidate);
    } else {
      console.warn("Unknown message received:", data);
    }
  };

  // Event handler for WebSocket errors
  signalingSocket.onerror = function (error) {
    console.error("Signaling server error:", error);
  };

  // Event handler for WebSocket connection close
  signalingSocket.onclose = function () {
    console.log("Disconnected from the signaling server");
  };
}

/**
 * Step 2: Initiate Session Establishment
 */
function initiateSession() {
  startButton.disabled = true;
  hangupButton.disabled = false;
  // Send SESSION request to the signaling server with the callee's UID
  signalingSocket.send(`SESSION ${calleeId}`);
  console.log(`Sent SESSION request to ${calleeId}`);
}

/**
 * Step 3: Handle the received SDP offer from the callee (GStreamer).
 * @param {string} sdpOffer - The SDP offer received as a JSON string.
 */
async function handleSdpOffer(sdpOffer) {
  console.log('Received SDP offer from callee');
  // Parse the SDP offer (it's a JSON string)
  const offer = JSON.parse(sdpOffer);

  // Create a new RTCPeerConnection with the specified configuration
  peerConnection = new RTCPeerConnection(configuration);
  setupPeerConnectionEventHandlers();

  await peerConnection.setRemoteDescription(new RTCSessionDescription(offer));
  console.log('Set remote description with SDP offer');
  remoteDescriptionSet = true;

  // Add any queued remote ICE candidates
  for (const candidate of remoteIceCandidatesQueue) {
    await peerConnection.addIceCandidate(candidate);
    console.log('Added queued remote ICE candidate');
  }
  remoteIceCandidatesQueue = [];

  // Create an SDP answer in response to the received SDP offer
  const answerDescription = await peerConnection.createAnswer();
  console.log('Created SDP answer');

  // Set the local description with the generated SDP answer
  await peerConnection.setLocalDescription(answerDescription);
  console.log('Set local description with SDP answer');

  // Prepare the SDP answer message with only 'type' and 'sdp'
  const sdpAnswer = {
    type: peerConnection.localDescription.type,
    sdp: peerConnection.localDescription.sdp,
  };

  // Send SDP_ANSWER back to the callee via the signaling server
  const sdpAnswerMessage = `SDP_ANSWER ${JSON.stringify(sdpAnswer)}`;
  signalingSocket.send(sdpAnswerMessage);
  console.log('Sent SDP_ANSWER to callee:', sdpAnswerMessage);
}

/**
 * Step 3 (Alternative): Handle the received SDP answer from the callee (if Client A was the offerer)
 * @param {string} sdpAnswer - The SDP answer received as a JSON string.
 */
async function handleSdpAnswer(sdpAnswer) {
  console.log('Received SDP answer from callee');

  // Parse the SDP answer (it's a JSON string)
  const answer = JSON.parse(sdpAnswer);

  // Set the remote description with the received SDP answer
  await peerConnection.setRemoteDescription(new RTCSessionDescription(answer));
  console.log('Set remote description with SDP answer');

  // Remote description is now set
  remoteDescriptionSet = true;

  // Add any queued remote ICE candidates that were received before the remote description was set
  for (const candidate of remoteIceCandidatesQueue) {
    await peerConnection.addIceCandidate(candidate);
    console.log('Added queued remote ICE candidate');
  }
  // Clear the queue after processing
  remoteIceCandidatesQueue = [];
}

/**
 * Set up event handlers for the RTCPeerConnection instance.
 */
function setupPeerConnectionEventHandlers() {
  peerConnection.onicecandidate = (event) => {
    if (event.candidate) {
      // Prepare the ICE candidate message with only 'candidate' and 'sdpMLineIndex'
      const candidate = {
        candidate: event.candidate.candidate,
        sdpMLineIndex: event.candidate.sdpMLineIndex,
      };

      // Send the ICE candidate to the remote peer via the signaling server
      const iceCandidateMessage = `ICE_CANDIDATE ${JSON.stringify(candidate)}`;
      signalingSocket.send(iceCandidateMessage);
      console.log('Sent local ICE candidate to callee:', iceCandidateMessage);
    } else {
      console.log('All local ICE candidates have been sent');
    }
  };

  // Event handler for receiving remote media tracks
  peerConnection.ontrack = (event) => {
    // Get the remote stream from the event
    const remoteStream = event.streams[0];
    console.log('Received remote media stream');

    // Display the remote video stream on the page
    if (remoteVideo) {
      remoteVideo.srcObject = remoteStream;
    }
  };

  // Event handler for changes in the ICE connection state
  peerConnection.oniceconnectionstatechange = () => {
    console.log('ICE connection state:', peerConnection.iceConnectionState);
    if (peerConnection.iceConnectionState === 'disconnected') {
      console.log('Peer disconnected');
      // Handle disconnection logic if necessary
      cleanUp();
    }
  };

  // Event handler for changes in the overall connection state
  peerConnection.onconnectionstatechange = () => {
    console.log('Peer connection state:', peerConnection.connectionState);
    if (peerConnection.connectionState === 'connected') {
      console.log('Peer connection established');
    } else if (
      peerConnection.connectionState === 'failed' ||
      peerConnection.connectionState === 'disconnected'
    ) {
      console.log('Peer connection closed or failed');
      // Handle failure logic if necessary
      cleanUp();
    }
  };
}

/**
 * Step 4: Handle the remote ICE candidate received from the callee.
 * @param {string} candidateData - The ICE candidate data received as a JSON string.
 */
async function handleRemoteIceCandidate(candidateData) {
  console.log('Received remote ICE candidate:', candidateData);
  // Parse the ICE candidate data
  const candidateObj = JSON.parse(candidateData);

  try {
    const iceCandidate = new RTCIceCandidate(candidateObj);
    if (remoteDescriptionSet) {
      // If the remote description is already set, add the ICE candidate immediately
      await peerConnection.addIceCandidate(iceCandidate);
      console.log('Added remote ICE candidate');
    } else {
      // Otherwise, queue the ICE candidate until the remote description is set
      remoteIceCandidatesQueue.push(iceCandidate);
      console.log('Queued remote ICE candidate');
    }
  } catch (error) {
    console.error('Error adding remote ICE candidate:', error);
  }
}

/**
 * Step 5: Establish Peer-to-Peer Connection
 * This step is inherently handled by the RTCPeerConnection and the ICE negotiation process.
 * Media streams (audio/video) flow directly between the peers once the connection is established.
 */
function cleanUp() {
  console.log('Hanging up the call');

  // Close the peer connection if it exists
  if (peerConnection) {
    peerConnection.close();
    peerConnection = null;
    console.log('Closed RTCPeerConnection');
  }

  // Reset video element
  if (remoteVideo) {
    remoteVideo.srcObject = null;
  }

  // Disable the hangup button and enable the start button
  hangupButton.disabled = true;
  startButton.disabled = false;

  // Optionally, notify the signaling server about the termination
  // signalingSocket.send(`TERMINATE ${uid}`); // Implement if needed
}

/**
 * Attach event listeners to control buttons
 */
startButton.addEventListener('click', () => {
  initiateSession();
});

hangupButton.addEventListener('click', () => {
  cleanUp();
});

/**
 * Start the connection process when the page loads.
 */
window.onload = () => {
  // Step 1: Establish WebSocket Connection and Handshake
  connectToSignalingServer();
};

I run first the signaling server,

./simple_server.py --disable-ssl

Starting server...
Listening on https://:8443
server listening on [::]:8443
server listening on 0.0.0.0:8443

then I start the GStreamer app:

Sent HELLO: HELLO drone
Received message: HELLO
Received HELLO from server

It succesfully connect, so I start the frontend with a python webserver:

Connected to the signaling server webrtc_client.js:33:13
Received message: HELLO webrtc_client.js:41:13
Received HELLO from server

It connects as well, I click on start call:

Sent SESSION request to drone webrtc_client.js:92:11
Received message: SESSION_OK webrtc_client.js:41:13
Received SESSION_OK from server webrtc_client.js:55:15
Received message: SDP_OFFER {"type":"offer","sdp":"v=0\r\no=- 
.....

All local ICE candidates have been sent webrtc_client.js:182:15
ICE connection state: checking webrtc_client.js:200:13
Peer connection state: connecting webrtc_client.js:210:13
ICE connection state: connected webrtc_client.js:200:13
Peer connection state: connected webrtc_client.js:210:13
Peer connection established webrtc_client.js:212:15

It connects to a session, sends SDP + Ice Candidates and I get that the peer connection is established. But on the GStreamer side I received empty candidate:

Set remote description with SDP answer
Processed SDP_ANSWER from browser
Received message: ICE_CANDIDATE {"candidate":"candidate:0 1 UDP 2122252543 172.17.52.39 37869 typ host","sdpMLineIndex":0}
Handling ICE_CANDIDATE: {"candidate":"candidate:0 1 UDP 2122252543 172.17.52.39 37869 typ host","sdpMLineIndex":0}
Received ICE candidate JSON: {"candidate":"candidate:0 1 UDP 2122252543 172.17.52.39 37869 typ host","sdpMLineIndex":0}
Added ICE_CANDIDATE from browser: candidate:0 1 UDP 2122252543 172.17.52.39 37869 typ host
Received message: ICE_CANDIDATE {"candidate":"candidate:1 1 TCP 2105524479 172.17.52.39 9 typ host tcptype active","sdpMLineIndex":0}
Handling ICE_CANDIDATE: {"candidate":"candidate:1 1 TCP 2105524479 172.17.52.39 9 typ host tcptype active","sdpMLineIndex":0}
Received ICE candidate JSON: {"candidate":"candidate:1 1 TCP 2105524479 172.17.52.39 9 typ host tcptype active","sdpMLineIndex":0}
Added ICE_CANDIDATE from browser: candidate:1 1 TCP 2105524479 172.17.52.39 9 typ host tcptype active
Received message: ICE_CANDIDATE {"candidate":"","sdpMLineIndex":0}
Handling ICE_CANDIDATE: {"candidate":"","sdpMLineIndex":0}
Received ICE candidate JSON: {"candidate":"","sdpMLineIndex":0}
Received empty ICE candidate, ignoring

And then nothing else happens and I get no stream on my web browser. Could someone point me out to what I am missing?
Totally lost for 3 days …

EDIT: it connects after a long time, but the quality is bad, the latency is terrible…

You should really be using the latest GStreamer 1.24.x series for webrtc things. There have been countless fixes in the WebRTC handling since 1.16.

Bad quality and bad latency may indicate bad network and may require handling network congestion by reducing bitrate. The webrtcsink element can do this for you automatically but is also possible using webrtcbin with some extra code.

it looks like you are using a software encoder (vp8enc) which won’t perform well on a jetson. the jetson should have a hardware encoder that you should use instead. that should help quite a bit