CC708Overlay to display closed caption on videosink

jignesh0103 · January 3, 2025, 12:22pm

I have developed gstreamer pipeline as following

#include <gst/gst.h>
#include <gst/app/gstappsink.h>
#include <iostream>
#include <vector>
#include <string>
#include <iomanip>

// This will hold the extracted captions
std::vector<std::string> closedCaptions;

static GstFlowReturn on_new_sample(GstAppSink *appsink, gpointer user_data) {
    // Retrieve the sample from the appsink
    GstSample *sample = gst_app_sink_pull_sample(appsink);
    if (sample) {
        // Extract the GstBuffer from the sample
        GstBuffer *buffer = gst_sample_get_buffer(sample);
        if (buffer) {
            guint size = gst_buffer_get_size(buffer);
            guint8 *data;
            GstMapInfo map_info;
            if (gst_buffer_map(buffer, &map_info, GST_MAP_READ)) {
                // Here, you would extract and process the closed caption data.
                std::vector<uint8_t> captionData(map_info.data, map_info.data + map_info.size);
                // Example output (debugging purposes)
                std::cout << "Caption Data (Hex): "<< map_info.size << std::endl;
                for (size_t i = 0; i < captionData.size(); ++i) {
                    std::cout << "0x" << std::hex << std::setw(2) << std::setfill('0') << (int)captionData[i] << " ";
                }
                std::cout << std::endl;
#if 0
                // Now, we need to send this caption data to the cc708overlay
                GstBuffer *caption_buffer = gst_buffer_new_and_alloc(captionData.size());
                gst_buffer_fill(caption_buffer, 0, captionData.data(), captionData.size());

                // Get the cc708overlay element from the pipeline
                GstElement *overlay_element = gst_bin_get_by_name(GST_BIN(user_data), "overlay");
                if (overlay_element) {
                    // Send the caption buffer to cc708overlay
                    GstPad *sink_pad = gst_element_get_static_pad(overlay_element, "sink");
                    if (GST_PAD_IS_LINKED(sink_pad)) {
                        GstPad *peer_pad = gst_pad_get_peer(sink_pad);
                        if (peer_pad) {
                            // Push the caption data to the sink pad
                            gst_pad_push(peer_pad, caption_buffer);
                            gst_object_unref(peer_pad);
                        }
                    }
                    gst_object_unref(sink_pad);
                }
#endif
                // Unmap the buffer after processing
                gst_buffer_unmap(buffer, &map_info);
            }
        }
        gst_sample_unref(sample);
    }
    return GST_FLOW_OK;
}

int main(int argc, char *argv[]) {
    gst_init(&argc, &argv);

GstElement *pipeline = gst_parse_launch(
    "udpsrc address=239.0.0.1 port=1234 buffer-size=100000000 ! tee name=t "
    "t. ! queue ! decodebin name=dmux "
    "dmux. ! queue ! audioconvert ! audioresample ! autoaudiosink "
    "dmux. ! queue ! videoconvert ! queue ! cc708overlay name=overlay ! queue !  autovideosink "
    "t. ! queue ! parsebin name=parser ! ccextractor name=extractor extractor.src ! cc708overlay.name=overlay cc_sink extractor.caption ! queue ! appsink name=caption_sink", NULL);

#if 0
    // Create the pipeline from the gst-launch command
    GstElement *pipeline = gst_parse_launch(
        "udpsrc address=239.0.0.1 port=1234 buffer-size=100000000 ! tee name=t "
        "t. ! queue ! decodebin name=dmux "
        "dmux. ! queue ! audioconvert ! audioresample ! autoaudiosink "
        "dmux. ! queue ! overlay.video_sink extractor.caption ! overlay.cc_sink cc708overlay name=overlay ! videoconvert ! autovideosink async=false"
        "t. ! queue ! parsebin name=parser ! ccextractor name=extractor extractor.src", NULL);
#endif
    // Retrieve the appsink for closed caption data
    GstElement *appsink = gst_bin_get_by_name(GST_BIN(pipeline), "caption_sink");
    if (!appsink) {
        std::cerr << "Error: Unable to retrieve appsink for captions!" << std::endl;
        return -1;
    }
    // Retrieve the cc708overlay element
    GstElement *overlay = gst_bin_get_by_name(GST_BIN(pipeline), "overlay");
    if (!overlay) {
        std::cerr << "Error: Unable to retrieve cc708overlay element!" << std::endl;
        return -1;
    }

    // Configure appsink to handle new samples
    g_object_set(appsink, "emit-signals", TRUE, NULL);
    g_signal_connect(appsink, "new-sample", G_CALLBACK(on_new_sample), pipeline);

    // Start the pipeline
    gst_element_set_state(pipeline, GST_STATE_PLAYING);

    // Wait until error or EOS (End Of Stream)
    GstBus *bus = gst_element_get_bus(pipeline);
    GstMessage *msg = NULL;
    gboolean running = TRUE;

    while (running) {
        msg = gst_bus_poll(bus, (GstMessageType)(GST_MESSAGE_ERROR | GST_MESSAGE_EOS), GST_CLOCK_TIME_NONE);
        if (msg != NULL) {
            GError *err = NULL;
            gchar *debug_info = NULL;

            switch (GST_MESSAGE_TYPE(msg)) {
                case GST_MESSAGE_ERROR:
                    gst_message_parse_error(msg, &err, &debug_info);
                    std::cerr << "Error: " << err->message << std::endl;
                    g_error_free(err);
                    g_free(debug_info);
                    running = FALSE;
                    break;
                case GST_MESSAGE_EOS:
                    std::cout << "End of Stream reached." << std::endl;
                    running = FALSE;
                    break;
                default:
                    break;
            }
            gst_message_unref(msg);
        }
    }

    // Clean up
    gst_object_unref(bus);
    gst_element_set_state(pipeline, GST_STATE_NULL);
    gst_object_unref(pipeline);

    return 0;
}

With above pipeline. I see that valid closed caption dump on app sink like following.

Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfd 0xce 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfd 0x43 0xce 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfd 0x8f 0x8c 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 
Caption Data (Hex): 1e

But on video I do not see any text overlay over video. I tried to enable full log for closed caption overlay but did not help much. Any help here to find any issue or exact reason why we do not able to see closed caption text over video is much helpful.

slomo · January 4, 2025, 8:33am

Why don’t you pass caption data directly to the overlay but instead go via the appsink? Doing it via the appsink requires a lot of care to make sure the formats and timing are matching up.

Also please try with cea708overlay, which supports both CEA608 and CEA708 captions, and works correctly. cc708overlay has lots of problems and is going to be removed.

ystreet00 · January 5, 2025, 12:23pm

Your caption data looks weird.

cc708overlay will not look at the 608 compatibility bytes. The caption data does not have any usable 708 data. Thus cc708overlay will not display anything.
The 608 data looks to be weird and non-standard: We have the following sequence of valid byte pairs for field 2: 0xce 0x80 | 0x43 0xce | 0x8f 0x8c The only value in that sequence that actually is actionable is the 0x43 which is ‘C’. The 0x8f 0x8c byte pair looks like an XDS ending packet which is not supported and will be ignored. All of the other values have no valid representation in CEA-608. Field 1 has no usable bytes.

jignesh0103 · January 7, 2025, 8:26am

Hi Matthew,

Thanks for checking but I think when I have took logs then there might be silent closed caption meaning no data… and during continue testing I see data there which should display text on video but it is not displaying…
Caption Data (Hex): 30
0xfc 0x94 0x25 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfc 0x91 0xe0 0xff 0x4a 0x31 0xfe 0x98 0x3b 0xfe 0x00 0x00 0xfe 0x01 0x1f 0xfe 0x10 0x90 0xfe 0x05 0x00 0xfe 0x91 0x2a 0xfe 0x00 0x00 0xfe 0x92 0x01
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfe 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfc 0x94 0xad 0xff 0x0b 0x32 0xfe 0x98 0x3b 0xfe 0x00 0x00 0xfe 0x01 0x1f 0xfe 0x10 0x90 0xfe 0x05 0x00 0xfe 0x91 0x2a 0xfe 0x00 0x00 0xfe 0x92 0x01
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfe 0x1b 0x0d 0xfe 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfc 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfd 0x80 0x80 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00
Caption Data (Hex): 1e
0xfc 0x4f 0x52 0xff 0x03 0x22 0xfe 0x4f 0x52 0xfe 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00 0xfa 0x00 0x00

Here I am attaching some data where I can see that it having valid data…

jignesh0103 · January 7, 2025, 8:27am

Hi Sebastian,

Thank you for response we are using buildroot mechanism for this but I will investigate about how to enable this cea708overlay plugging.